Migrate apps to Spring Functions Catalog (#523)
* Remove `functions` module * Remove `functions` from build scripts * Change internal `functions` mentioning in the docs to the dedication Spring Functions Catalog project * Remove all the TensorFlow modules * Change to `spring-functions-catalog-bom` * Update READMEs using `quick-compile.sh`
This commit is contained in:
Artem Bilan
107
README.adoc
107
README.adoc
@@ -2,114 +2,20 @@
|
||||
|
||||
In this repository, you will find a collection of components that can meet various data integration use cases and requirements.
|
||||
|
||||
The repository's primary focus is to provide a set of standalone Java functions that can be useful in the end-user applications as-is.
|
||||
|
||||
Besides, this repository builds on the Java functions to generate standalone Spring Cloud Stream applications that can run against Spring Cloud Stream's RabbitMQ or Apache Kafka binder implementations.
|
||||
This repository builds on top of the https://github.com/spring-cloud/spring-functions-catalog[Spring Functions Catalog] and generates standalone Spring Cloud Stream applications that can run against Spring Cloud Stream's RabbitMQ or Apache Kafka binder implementations.
|
||||
It is also possible to extend the generator to bundle the Java functions with the other supported binder implementations.
|
||||
|
||||
These applications can run standalone or as part of a data flow, such as the one orchestrated using Spring Cloud Data Flow.
|
||||
|
||||
=== Project Structure
|
||||
|
||||
The repository includes two major sections - `Functions` and `Applications`.
|
||||
The former hosts the various Java functions, and the latter is for generating the standalone Spring Cloud Stream applications and hosting their related components.
|
||||
|
||||
The following are the various components of this repository.
|
||||
|
||||
* https://github.com/spring-cloud/stream-applications/tree/master/functions[Standalone Java Functions]
|
||||
* https://github.com/spring-cloud/stream-applications/tree/master/applications/stream-applications-core[Common Core Components for Applications]
|
||||
* https://github.com/spring-cloud/stream-applications/tree/master/applications[Spring Cloud Stream Applications]
|
||||
* https://github.com/spring-cloud/stream-applications/tree/master/stream-applications-build[Build Parent]
|
||||
* https://github.com/spring-cloud/stream-applications/tree/master/stream-applications-release-train[Release Train]
|
||||
|
||||
=== Reusable Functions
|
||||
|
||||
|===
|
||||
| `java.util.Supplier` | `java.util.Function` | `java.util.Consumer`
|
||||
|
||||
|link:functions/supplier/debezium-supplier/README.adoc[Debezium supplier]
|
||||
|link:functions/function/aggregator-function/README.adoc[Aggregator]
|
||||
|link:functions/consumer/analytics-consumer/README.adoc[Analytics]
|
||||
|
||||
|link:functions/supplier/file-supplier/README.adoc[File]
|
||||
|link:functions/function/filter-function/README.adoc[Filter]
|
||||
|link:functions/consumer/cassandra-consumer/README.adoc[Cassandra]
|
||||
|
||||
|link:functions/supplier/ftp-supplier/README.adoc[FTP]
|
||||
|link:functions/function/header-enricher-function/README.adoc[Header-Enricher]
|
||||
|link:functions/consumer/elasticsearch-consumer/README.adoc[Elasticsearch]
|
||||
|
||||
|
|
||||
|link:functions/function/header-filter-function/README.adoc[Header-Filter]
|
||||
|link:functions/consumer/file-consumer/README.adoc[File]
|
||||
|
||||
|link:functions/supplier/http-supplier/README.adoc[HTTP]
|
||||
|link:functions/function/http-request-function/README.adoc[HTTP Request]
|
||||
|link:functions/consumer/ftp-consumer/README.adoc[FTP]
|
||||
|
||||
|link:functions/supplier/jdbc-supplier/README.adoc[JDBC]
|
||||
|link:functions/function/image-recognition-function/README.adoc[Image Recognition(Tensorflow)]
|
||||
|
|
||||
|
||||
|link:functions/supplier/jms-supplier/README.adoc[JMS]
|
||||
|link:functions/function/object-detection-function/README.adoc[Object Detection(Tensorflow)]
|
||||
|link:functions/consumer/jdbc-consumer/README.adoc[JDBC]
|
||||
|
||||
|link:functions/supplier/mail-supplier/README.adoc[Mail]
|
||||
|link:functions/function/semantic-segmentation-function/README.adoc[Semantic Segmentation(Tensorflow)]
|
||||
|link:functions/consumer/log-consumer/README.adoc[Log]
|
||||
|
||||
|link:functions/supplier/mongodb-supplier/README.adoc[MongoDB]
|
||||
|link:functions/function/spel-function/README.adoc[SpEL]
|
||||
|link:functions/consumer/mongodb-consumer/README.adoc[MongoDB]
|
||||
|
||||
|link:functions/supplier/mqtt-supplier/README.adoc[MQTT]
|
||||
|
||||
|link:functions/function/splitter-function/README.adoc[Splitter]
|
||||
|link:functions/consumer/mqtt-consumer/README.adoc[MQTT]
|
||||
|
||||
|link:functions/supplier/rabbit-supplier/README.adoc[RabbitMQ]
|
||||
|link:functions/function/task-launch-request-function/README.adoc[Task Launch Request]
|
||||
|link:functions/consumer/rabbit-consumer/README.adoc[RabbitMQ]
|
||||
|
||||
|link:functions/supplier/s3-supplier/README.adoc[AWS S3]
|
||||
|
|
||||
|link:functions/consumer/redis-consumer/README.adoc[Redis]
|
||||
|
||||
|link:functions/supplier/sftp-supplier/README.adoc[SFTP]
|
||||
|
|
||||
|link:functions/consumer/rsocket-consumer/README.adoc[RSocket]
|
||||
|
||||
|link:functions/supplier/syslog-supplier/README.adoc[Syslog]
|
||||
|
|
||||
|link:functions/consumer/s3-consumer/README.adoc[AWS S3]
|
||||
|
||||
|link:functions/supplier/tcp-supplier/README.adoc[TCP]
|
||||
|
|
||||
|link:functions/consumer/sftp-consumer/README.adoc[SFTP]
|
||||
|
||||
|link:functions/supplier/time-supplier/README.adoc[Time]
|
||||
|
|
||||
|link:functions/consumer/tcp-consumer/README.adoc[TCP]
|
||||
|
||||
|link:functions/supplier/twitter-supplier/README.adoc[Twitter]
|
||||
|link:functions/function/twitter-function/README.adoc[Twitter]
|
||||
|link:functions/consumer/twitter-consumer/README.adoc[Twitter]
|
||||
|
||||
|link:functions/supplier/websocket-supplier/README.adoc[Websocket]
|
||||
|
|
||||
|link:functions/consumer/websocket-consumer/README.adoc[Websocket]
|
||||
|
||||
|
|
||||
|
|
||||
|link:functions/consumer/wavefront-consumer/README.adoc[Wavefront]
|
||||
|
||||
|link:functions/supplier/xmpp-supplier/README.adoc[XMPP]
|
||||
|
|
||||
|link:functions/consumer/xmpp-consumer/README.adoc[XMPP]
|
||||
|
||||
|===
|
||||
|
||||
=== Reusable Spring Cloud Stream Applications
|
||||
|
||||
|===
|
||||
@@ -139,11 +45,11 @@ The following are the various components of this repository.
|
||||
|
|
||||
|
||||
|link:applications/source/jms-source/README.adoc[JMS]
|
||||
|link:applications/processor/image-recognition-processor/README.adoc[Image Recognition(Tensorflow)]
|
||||
|
|
||||
|link:applications/sink/jdbc-sink/README.adoc[JDBC]
|
||||
|
||||
|link:applications/source/load-generator-source/README.adoc[Load-Generator]
|
||||
|link:applications/processor/object-detection-processor/README.adoc[Object Detection(Tensorflow)]
|
||||
|
|
||||
|link:applications/sink/log-sink/README.adoc[Log]
|
||||
|
||||
|link:applications/source/mail-source/README.adoc[Mail]
|
||||
@@ -151,7 +57,7 @@ The following are the various components of this repository.
|
||||
|link:applications/sink/mongodb-sink/README.adoc[MongoDB]
|
||||
|
||||
|link:applications/source/mongodb-source/README.adoc[MongoDB]
|
||||
|link:applications/processor/semantic-segmentation-processor/README.adoc[Semantic Segmentation(Tensorflow)]
|
||||
|
|
||||
|link:applications/sink/mqtt-sink/README.adoc[MQTT]
|
||||
|
||||
|link:applications/source/mqtt-source/README.adoc[MQTT]
|
||||
@@ -180,7 +86,7 @@ The following are the various components of this repository.
|
||||
|
||||
|link:applications/source/time-source/README.adoc[Time]
|
||||
|
|
||||
|link:applications/sink/tasklauncher-sink/README.adoc[Task Launcher]
|
||||
|
|
||||
|
||||
|link:applications/source/twitter-message-source/README.adoc[Twitter Message]
|
||||
|
|
||||
@@ -253,7 +159,8 @@ You can then build the desired apps.
|
||||
./build-app.sh . applications/sink/log-sink
|
||||
....
|
||||
|
||||
NOTE: In order to disable metrics by default there needs to be application properties configured like in `default-application.properties`. The `build-app.sh` script will copy default-application.properties into src/main/resources if no application.properties,yml,yaml or json is present.
|
||||
NOTE: In order to disable metrics by default there needs to be application properties configured like in `default-application.properties`.
|
||||
The `build-app.sh` script will copy default-application.properties into src/main/resources if no application.properties,yml,yaml or json is present.
|
||||
|
||||
=== Additional Resources
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>aggregator-function</artifactId>
|
||||
<artifactId>spring-aggregator-function</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
@@ -35,13 +35,13 @@
|
||||
<name>aggregator</name>
|
||||
<type>processor</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.aggregator.AggregatorFunctionConfiguration.class</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>jsonBytesToMap|aggregatorFunction</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>aggregator-function</artifactId>
|
||||
<artifactId>spring-aggregator-function</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>filter-function</artifactId>
|
||||
<artifactId>spring-filter-function</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
@@ -43,13 +43,13 @@
|
||||
<name>filter</name>
|
||||
<type>processor</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.filter.FilterFunctionConfiguration.class</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>byteArrayTextToString|filterFunction</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>filter-function</artifactId>
|
||||
<artifactId>spring-filter-function</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -54,7 +54,7 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>payload-converter-function</artifactId>
|
||||
<artifactId>spring-payload-converter-function</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>header-enricher-function</artifactId>
|
||||
<artifactId>spring-header-enricher-function</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.integration</groupId>
|
||||
@@ -48,13 +48,13 @@
|
||||
<name>header-enricher</name>
|
||||
<type>processor</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.header.enricher.HeaderEnricherFunctionConfiguration.class</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>headerEnricherFunction</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>header-enricher-function</artifactId>
|
||||
<artifactId>spring-header-enricher-function</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>header-filter-function</artifactId>
|
||||
<artifactId>spring-header-filter-function</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.integration</groupId>
|
||||
@@ -53,13 +53,13 @@
|
||||
<name>header-filter</name>
|
||||
<type>processor</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.header.filter.HeaderFilterFunctionConfiguration.class</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>headerFilterFunction</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>header-filter-function</artifactId>
|
||||
<artifactId>spring-header-filter-function</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -52,7 +52,7 @@ $$body-expression$$:: $$A SpEL expression to derive the request body from the in
|
||||
$$expected-response-type$$:: $$The type used to interpret the response.$$ *($$Class<?>$$, default: `$$<none>$$`)*
|
||||
$$headers-expression$$:: $$A SpEL expression used to derive the http headers map to use.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$http-method-expression$$:: $$A SpEL expression to derive the request method from the incoming message.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$reply-expression$$:: $$A SpEL expression used to compute the final result, applied against the whole http {@link org.springframework.http.ResponseEntity}.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$reply-expression$$:: $$A SpEL expression used to compute the final result, applied against the whole http {@link ResponseEntity}.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$timeout$$:: $$Request timeout in milliseconds.$$ *($$Long$$, default: `$$30000$$`)*
|
||||
$$url-expression$$:: $$A SpEL expression against incoming message to determine the URL to use.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>http-request-function</artifactId>
|
||||
<artifactId>spring-http-request-function</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.squareup.okhttp3</groupId>
|
||||
@@ -56,15 +56,14 @@
|
||||
<name>http-request</name>
|
||||
<type>processor</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.http.request.HttpRequestFunctionConfiguration.class
|
||||
</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>httpRequestFunction</functionDefinition>
|
||||
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>http-request-function</artifactId>
|
||||
<artifactId>spring-http-request-function</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
//tag::ref-doc[]
|
||||
:image-root: https://raw.githubusercontent.com/spring-cloud-stream-app-starters/tensorflow/master/images
|
||||
|
||||
= Image Recognition Processor
|
||||
|
||||
A processor that uses an https://github.com/tensorflow/models/tree/master/inception[Inception model] to classify
|
||||
in real-time images into different categories (e.g. labels).
|
||||
|
||||
Model implements a deep https://en.wikipedia.org/wiki/Convolutional_neural_network[Convolutional Neural Network] that can achieve reasonable performance on hard visual recognition tasks
|
||||
- matching or exceeding human performance in some domains like https://www.tensorflow.org/tutorials/image_recognition[image recognition].
|
||||
|
||||
The input of the model is an image as binary array.
|
||||
|
||||
The output is a JSON message in this format:
|
||||
|
||||
[source,json]
|
||||
....
|
||||
{
|
||||
"labels" : [
|
||||
{"giant panda":0.98649305}
|
||||
]
|
||||
}
|
||||
....
|
||||
|
||||
Result contains the name of the recognized category (e.g. label) along with the confidence (e.g. confidence) that the image represents this category.
|
||||
|
||||
If the `response-seize` is set to value higher then 1, then the result will include the top `response-seize` probable labels. For example `response-size=3` would return:
|
||||
|
||||
[source,json]
|
||||
....
|
||||
{
|
||||
"labels": [
|
||||
{"giant panda":0.98649305},
|
||||
{"badger":0.010562794},
|
||||
{"ice bear":0.001130851}
|
||||
]
|
||||
}
|
||||
....
|
||||
|
||||
== Payload
|
||||
|
||||
If the incoming type is `byte[]` and the content type is set to `application/octet-stream` , then the application process the input `byte[]` image into and outputs augmented `byte[]` image payload and json header.
|
||||
|
||||
== Options
|
||||
|
||||
//tag::configuration-properties[]
|
||||
$$image.recognition.cache-model$$:: $$cache the pre-trained tensorflow model.$$ *($$Boolean$$, default: `$$<none>$$`)*
|
||||
$$image.recognition.debug-output$$:: $$<documentation missing>$$ *($$Boolean$$, default: `$$<none>$$`)*
|
||||
$$image.recognition.debug-output-path$$:: $$<documentation missing>$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$image.recognition.model$$:: $$pre-trained tensorflow image recognition model. Note that the model must match the selected model type!$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$image.recognition.model-type$$:: $$Supports three different pre-trained tensorflow image recognition models: Inception, MobileNetV1 and MobileNetV2 1. Inception graph uses 'input' as input and 'output' as output. 2. MobileNetV2 pre-trained models: https://github.com/tensorflow/models/tree/master/research/slim/nets/mobilenet#pretrained-models - normalized image size is always square (e.g. H=W) - graph uses 'input' as input and 'MobilenetV2/Predictions/Reshape_1' as output. 3. MobileNetV1 pre-trained models: https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md#pre-trained-models - graph uses 'input' as input and 'MobilenetV1/Predictions/Reshape_1' as output.$$ *($$ModelType$$, default: `$$<none>$$`, possible values: `inception`,`mobilenetv1`,`mobilenetv2`)*
|
||||
$$image.recognition.normalized-image-size$$:: $$Normalized image size.$$ *($$Integer$$, default: `$$<none>$$`)*
|
||||
$$image.recognition.response-size$$:: $$number of recognized images.$$ *($$Integer$$, default: `$$<none>$$`)*
|
||||
//end::configuration-properties[]
|
||||
|
||||
//end::ref-doc[]
|
||||
@@ -1,69 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>image-recognition-processor</artifactId>
|
||||
<name>image-recognition-processor</name>
|
||||
<description>Image recognition (tensorflow) processor apps</description>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<parent>
|
||||
<groupId>org.springframework.cloud.stream.app</groupId>
|
||||
<artifactId>stream-applications-core</artifactId>
|
||||
<version>5.0.0-SNAPSHOT</version>
|
||||
<relativePath>../../stream-applications-core/pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-configuration-processor</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>image-recognition-function</artifactId>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-deploy-plugin</artifactId>
|
||||
<version>3.0.0</version>
|
||||
<configuration>
|
||||
<skip>false</skip>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-dataflow-apps-docs-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-dataflow-apps-generator-plugin</artifactId>
|
||||
<configuration>
|
||||
<application>
|
||||
<name>image-recognition</name>
|
||||
<type>processor</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.stream.app.processor.image.recognition.ImageRecognitionProcessorConfiguration.class</configClass>
|
||||
<functionDefinition>imageRecognitionFunction</functionDefinition>
|
||||
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.stream.app</groupId>
|
||||
<artifactId>image-recognition-processor</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
</application>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
@@ -1,114 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020-2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.app.processor.image.recognition;
|
||||
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.function.Function;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.fn.common.tensorflow.deprecated.JsonMapperFunction;
|
||||
import org.springframework.cloud.fn.image.recognition.ImageRecognition;
|
||||
import org.springframework.cloud.fn.image.recognition.ImageRecognitionAugmenter;
|
||||
import org.springframework.cloud.fn.image.recognition.RecognitionResponse;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.integration.support.MessageBuilder;
|
||||
import org.springframework.messaging.Message;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
*/
|
||||
@Configuration
|
||||
@EnableConfigurationProperties(ImageRecognitionProcessorProperties.class)
|
||||
public class ImageRecognitionProcessorConfiguration {
|
||||
|
||||
private static final Log logger = LogFactory.getLog(ImageRecognitionProcessorConfiguration.class);
|
||||
|
||||
/**
|
||||
* Name of the Message header containing the JSON encoded recognition response.
|
||||
*/
|
||||
public static final String RECOGNIZED_OBJECTS_HEADER = "recognized_objects";
|
||||
|
||||
@Bean
|
||||
public Function<Message<byte[]>, Message<byte[]>> imageRecognitionFunction(ImageRecognitionProcessorProperties properties) {
|
||||
|
||||
return input -> {
|
||||
// You can use file:, http: or classpath: to provide the path to the input image.
|
||||
byte[] inputImage = input.getPayload();
|
||||
|
||||
try (ImageRecognition imageRecognition = createImageRecognitionFunction(properties)) {
|
||||
|
||||
List<RecognitionResponse> recognizedObjects =
|
||||
ImageRecognition.toRecognitionResponse(imageRecognition.recognizeTopK(inputImage));
|
||||
|
||||
// Draw the predicted labels on top of the input image.
|
||||
byte[] augmentedImage = new ImageRecognitionAugmenter().apply(inputImage, recognizedObjects);
|
||||
|
||||
String jsonRecognizedObjects = new JsonMapperFunction().apply(recognizedObjects);
|
||||
|
||||
Message<byte[]> outMessage = MessageBuilder
|
||||
.withPayload(augmentedImage)
|
||||
.setHeader(RECOGNIZED_OBJECTS_HEADER, jsonRecognizedObjects)
|
||||
.build();
|
||||
|
||||
if (properties.isDebugOutput()) {
|
||||
try {
|
||||
logger.info("recognized objects = " + jsonRecognizedObjects);
|
||||
IOUtils.write(augmentedImage, new FileOutputStream(properties.getDebugOutputPath()));
|
||||
}
|
||||
catch (IOException e) {
|
||||
logger.warn("Cloud not produce debug output", e);
|
||||
}
|
||||
}
|
||||
|
||||
return outMessage;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static ImageRecognition createImageRecognitionFunction(ImageRecognitionProcessorProperties properties) {
|
||||
switch (properties.getModelType()) {
|
||||
case inception:
|
||||
return ImageRecognition.inception(
|
||||
properties.getModel(),
|
||||
properties.getNormalizedImageSize(),
|
||||
properties.getResponseSize(),
|
||||
properties.isCacheModel());
|
||||
case mobilenetv1:
|
||||
return ImageRecognition.mobileNetV1(
|
||||
properties.getModel(),
|
||||
properties.getNormalizedImageSize(),
|
||||
properties.getResponseSize(),
|
||||
properties.isCacheModel());
|
||||
case mobilenetv2:
|
||||
return ImageRecognition.mobileNetV2(
|
||||
properties.getModel(),
|
||||
properties.getNormalizedImageSize(),
|
||||
properties.getResponseSize(),
|
||||
properties.isCacheModel());
|
||||
default:
|
||||
throw new RuntimeException("Not supported Model Type: " + properties.getModelType());
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,126 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020-2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.app.processor.image.recognition;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
*/
|
||||
@ConfigurationProperties("image.recognition")
|
||||
@Validated
|
||||
public class ImageRecognitionProcessorProperties {
|
||||
|
||||
enum ModelType {
|
||||
inception,
|
||||
mobilenetv1,
|
||||
mobilenetv2
|
||||
}
|
||||
|
||||
/**
|
||||
* Supports three different pre-trained tensorflow image recognition models: Inception, MobileNetV1 and MobileNetV2
|
||||
*
|
||||
* 1. Inception graph uses 'input' as input and 'output' as output.
|
||||
* 2. MobileNetV2 pre-trained models: https://github.com/tensorflow/models/tree/master/research/slim/nets/mobilenet#pretrained-models
|
||||
* - normalized image size is always square (e.g. H=W)
|
||||
* - graph uses 'input' as input and 'MobilenetV2/Predictions/Reshape_1' as output.
|
||||
* 3. MobileNetV1 pre-trained models: https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md#pre-trained-models
|
||||
* - graph uses 'input' as input and 'MobilenetV1/Predictions/Reshape_1' as output.
|
||||
*/
|
||||
private ModelType modelType = ModelType.mobilenetv2;
|
||||
|
||||
/**
|
||||
* pre-trained tensorflow image recognition model. Note that the model must match the selected model type!
|
||||
*/
|
||||
private String model = "https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.4_224.tgz#mobilenet_v2_1.4_224_frozen.pb";
|
||||
|
||||
/**
|
||||
* cache the pre-trained tensorflow model.
|
||||
*/
|
||||
private boolean cacheModel = true;
|
||||
|
||||
/**
|
||||
* Normalized image size.
|
||||
*/
|
||||
private int normalizedImageSize = 224;
|
||||
|
||||
/**
|
||||
* number of recognized images.
|
||||
*/
|
||||
private int responseSize = 5;
|
||||
|
||||
private boolean debugOutput = false;
|
||||
|
||||
private String debugOutputPath = "image-recognition-result.png";
|
||||
|
||||
public ModelType getModelType() {
|
||||
return modelType;
|
||||
}
|
||||
|
||||
public void setModelType(ModelType modelType) {
|
||||
this.modelType = modelType;
|
||||
}
|
||||
|
||||
public String getModel() {
|
||||
return model;
|
||||
}
|
||||
|
||||
public void setModel(String model) {
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
public boolean isCacheModel() {
|
||||
return cacheModel;
|
||||
}
|
||||
|
||||
public void setCacheModel(boolean cacheModel) {
|
||||
this.cacheModel = cacheModel;
|
||||
}
|
||||
|
||||
public int getNormalizedImageSize() {
|
||||
return normalizedImageSize;
|
||||
}
|
||||
|
||||
public void setNormalizedImageSize(int normalizedImageSize) {
|
||||
this.normalizedImageSize = normalizedImageSize;
|
||||
}
|
||||
|
||||
public int getResponseSize() {
|
||||
return responseSize;
|
||||
}
|
||||
|
||||
public void setResponseSize(int responseSize) {
|
||||
this.responseSize = responseSize;
|
||||
}
|
||||
|
||||
public boolean isDebugOutput() {
|
||||
return debugOutput;
|
||||
}
|
||||
|
||||
public void setDebugOutput(boolean debugOutput) {
|
||||
this.debugOutput = debugOutput;
|
||||
}
|
||||
|
||||
public String getDebugOutputPath() {
|
||||
return debugOutputPath;
|
||||
}
|
||||
|
||||
public void setDebugOutputPath(String debugOutputPath) {
|
||||
this.debugOutputPath = debugOutputPath;
|
||||
}
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
configuration-properties.classes=\
|
||||
org.springframework.cloud.stream.app.processor.image.recognition.ImageRecognitionProcessorProperties
|
||||
@@ -1,16 +0,0 @@
|
||||
image:
|
||||
recognition:
|
||||
model: https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.4_224.tgz#mobilenet_v2_1.4_224_frozen.pb
|
||||
modelType: mobilenetv2
|
||||
responseSize: 3
|
||||
normalizedImageSize: 224
|
||||
cacheModel: true
|
||||
spring:
|
||||
cloud:
|
||||
function:
|
||||
definition: imageRecognitionFunction
|
||||
management:
|
||||
defaults:
|
||||
metrics:
|
||||
export:
|
||||
enabled: false
|
||||
@@ -1,190 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020-2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.app.processor.image.recognition;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.math.BigDecimal;
|
||||
import java.math.RoundingMode;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import org.springframework.boot.WebApplicationType;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
import org.springframework.cloud.fn.common.tensorflow.deprecated.GraphicsUtils;
|
||||
import org.springframework.cloud.stream.binder.test.InputDestination;
|
||||
import org.springframework.cloud.stream.binder.test.OutputDestination;
|
||||
import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.messaging.support.GenericMessage;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
*/
|
||||
public class ImageRecognitionProcessorTests {
|
||||
|
||||
private ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Test
|
||||
public void testImageRecognitionProcessorMobileNetV2() throws IOException {
|
||||
List<Map<String, Object>> expected = deserializeAndRoundToNPlaces(
|
||||
"[{\"label\":\"giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca\",\"probability\":0.962329626083374},"
|
||||
+
|
||||
"{\"label\":\"badger\",\"probability\":0.006058811210095882}," +
|
||||
"{\"label\":\"ram, tup\",\"probability\":0.0010668420000001788}]",
|
||||
6);
|
||||
|
||||
imageRecognitionProcessorMobileNetV2(verify(expected));
|
||||
}
|
||||
|
||||
private void imageRecognitionProcessorMobileNetV2(Consumer<Message<byte[]>> consumer) throws IOException {
|
||||
try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
|
||||
TestChannelBinderConfiguration.getCompleteConfiguration(ImageRecognitionProcessorTestApplication.class))
|
||||
.web(WebApplicationType.NONE)
|
||||
.run("--spring.cloud.function.definition=imageRecognitionFunction",
|
||||
"--image.recognition.modelType=mobilenetv2",
|
||||
"--image.recognition.responseSize=3",
|
||||
"--image.recognition.debugOutput=true",
|
||||
"--image.recognition.debugOutputPath=./target/image-recognition-mobilenetv2.png")) {
|
||||
|
||||
InputDestination processorInput = context.getBean(InputDestination.class);
|
||||
OutputDestination processorOutput = context.getBean(OutputDestination.class);
|
||||
|
||||
byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/giant_panda_in_beijing_zoo_1.jpg");
|
||||
processorInput.send(new GenericMessage<>(inputImage));
|
||||
Message<byte[]> sourceMessage = processorOutput.receive(10000);
|
||||
consumer.accept(sourceMessage);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testImageRecognitionProcessorMobileNetV1() throws IOException {
|
||||
List<Map<String, Object>> expected = deserializeAndRoundToNPlaces(
|
||||
"[{\"label\":\"giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca\",\"probability\":0.984053909778595},"
|
||||
+
|
||||
"{\"label\":\"ram, tup\",\"probability\":0.0019619385711848736}," +
|
||||
"{\"label\":\"Staffordshire bullterrier, Staffordshire bull terrier\",\"probability\":0.0018697341438382864}]",
|
||||
6);
|
||||
|
||||
imageRecognitionProcessorMobileNetV1(verify(expected));
|
||||
}
|
||||
|
||||
private void imageRecognitionProcessorMobileNetV1(Consumer<Message<byte[]>> consumer) throws IOException {
|
||||
try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
|
||||
TestChannelBinderConfiguration.getCompleteConfiguration(ImageRecognitionProcessorTestApplication.class))
|
||||
.web(WebApplicationType.NONE)
|
||||
.run("--image.recognition.model=https://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224.tgz#mobilenet_v1_1.0_224_frozen.pb",
|
||||
"--image.recognition.modelType=mobilenetv1",
|
||||
"--image.recognition.responseSize=3",
|
||||
"--image.recognition.debugOutput=true",
|
||||
"--image.recognition.debugOutputPath=./target/image-recognition-mobilenetv1.png")) {
|
||||
|
||||
InputDestination processorInput = context.getBean(InputDestination.class);
|
||||
OutputDestination processorOutput = context.getBean(OutputDestination.class);
|
||||
|
||||
byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/giant_panda_in_beijing_zoo_1.jpg");
|
||||
processorInput.send(new GenericMessage<>(inputImage));
|
||||
Message<byte[]> sourceMessage = processorOutput.receive(10000);
|
||||
consumer.accept(sourceMessage);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testImageRecognitionProcessorInception() throws IOException {
|
||||
List<Map<String, Object>> expected = deserializeAndRoundToNPlaces(
|
||||
"[{\"label\":\"giant panda\",\"probability\":0.9946685433387756}," +
|
||||
"{\"label\":\"Arctic fox\",\"probability\":0.003663112409412861}," +
|
||||
"{\"label\":\"ice bear\",\"probability\":3.378273395355791E-4}]",
|
||||
6);
|
||||
imageRecognitionProcessorInception(verify(expected));
|
||||
}
|
||||
|
||||
private void imageRecognitionProcessorInception(Consumer<Message<byte[]>> consumer) throws IOException {
|
||||
try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
|
||||
TestChannelBinderConfiguration.getCompleteConfiguration(ImageRecognitionProcessorTestApplication.class))
|
||||
.web(WebApplicationType.NONE)
|
||||
.run("--image.recognition.model=https://storage.googleapis.com/scdf-tensorflow-models/image-recognition/tensorflow_inception_graph.pb",
|
||||
"--image.recognition.modelType=inception",
|
||||
"--image.recognition.responseSize=3",
|
||||
"--image.recognition.debugOutput=true",
|
||||
"--image.recognition.debugOutputPath=./target/image-recognition-inception.png")) {
|
||||
|
||||
InputDestination processorInput = context.getBean(InputDestination.class);
|
||||
OutputDestination processorOutput = context.getBean(OutputDestination.class);
|
||||
|
||||
byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/giant_panda_in_beijing_zoo_1.jpg");
|
||||
processorInput.send(new GenericMessage<>(inputImage));
|
||||
Message<byte[]> sourceMessage = processorOutput.receive(10000);
|
||||
|
||||
consumer.accept(sourceMessage);
|
||||
}
|
||||
}
|
||||
|
||||
private Consumer<Message<byte[]>> verify(List<Map<String, Object>> expected) {
|
||||
return message -> {
|
||||
List<Map<String, Object>> actual = deserializeAndRoundToNPlaces((String) message.getHeaders()
|
||||
.get(ImageRecognitionProcessorConfiguration.RECOGNIZED_OBJECTS_HEADER), 6);
|
||||
assertThat(expected)
|
||||
.isEqualTo(actual);
|
||||
};
|
||||
}
|
||||
|
||||
private List<Map<String, Object>> deserializeAndRoundToNPlaces(String json, int places) {
|
||||
List<Map<String, Object>> result = null;
|
||||
try {
|
||||
result = objectMapper.readValue(json, ArrayList.class);
|
||||
}
|
||||
catch (JsonProcessingException e) {
|
||||
throw new IllegalStateException(e.getMessage(), e);
|
||||
}
|
||||
|
||||
result.forEach(map -> {
|
||||
if (map.containsKey("probability")) {
|
||||
map.put("probability", round((double) map.get("probability"), places));
|
||||
}
|
||||
});
|
||||
|
||||
return result;
|
||||
|
||||
}
|
||||
|
||||
private static double round(double value, int places) {
|
||||
if (places < 0) {
|
||||
throw new IllegalArgumentException();
|
||||
}
|
||||
|
||||
BigDecimal bd = new BigDecimal(Double.toString(value));
|
||||
bd = bd.setScale(places, RoundingMode.HALF_UP);
|
||||
return bd.doubleValue();
|
||||
}
|
||||
|
||||
@SpringBootApplication
|
||||
@Import({ ImageRecognitionProcessorConfiguration.class })
|
||||
public static class ImageRecognitionProcessorTestApplication {
|
||||
}
|
||||
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 113 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 11 KiB |
@@ -1,57 +0,0 @@
|
||||
//tag::ref-doc[]
|
||||
:image-root: https://github.com/spring-cloud/stream-applications/raw/master/applications/processor/object-detection-processor/src/test/resources/images
|
||||
|
||||
= Object Detection Processor
|
||||
|
||||
The Object Detection processor provides out-of-the-box support for the https://github.com/tensorflow/models/blob/master/research/object_detection/README.md[TensorFlow Object Detection API]. It allows for real-time localization and identification of multiple objects in a single image or image stream. The Object Detection processor is built on top of the https://github.com/spring-cloud/stream-applications/tree/master/functions/function/object-detection-function[Object Detection Function].
|
||||
|
||||
You have to provide the Processor with a pre-trained https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md[object detection model], and the corresponding https://github.com/tensorflow/models/tree/865c14c/research/object_detection/data[object labels].
|
||||
|
||||
Here are some sensible configuration defaults:
|
||||
|
||||
* `object.detection.model` : `https://storage.googleapis.com/scdf-tensorflow-models/object-detection/faster_rcnn_resnet101_coco_2018_01_28_frozen_inference_graph.pb`
|
||||
* `object.detection.labels` : `https://storage.googleapis.com/scdf-tensorflow-models/object-detection/mscoco_label_map.pbtxt`
|
||||
* `object.detection.with-masks` : `false`
|
||||
|
||||
The following diagram shows a https://dataflow.spring.io/docs/concepts/streams/[Spring Cloud Data Flow], streaming pipeline, that predicts, in real-time, the object types in input image stream.
|
||||
|
||||
image::{image-root}/scdf-tensorflow-object-detection-arch.png[]
|
||||
|
||||
Processor's input is an image byte array, and the output is an augmented image, and a header, called `detected_objects`, that provides textual description of the detected objects:
|
||||
|
||||
[source,json]
|
||||
....
|
||||
{
|
||||
"labels" : [
|
||||
{"name":"person", "confidence":0.9996774,"x1":0.0,"y1":0.3940161,"x2":0.9465165,"y2":0.5592592,"cid":1},
|
||||
{"name":"person", "confidence":0.9996604,"x1":0.047891676,"y1":0.03169123,"x2":0.941098,"y2":0.2085562,"cid":1},
|
||||
{"name":"backpack", "confidence":0.96534747,"x1":0.15588468,"y1":0.85957795,"x2":0.5091308,"y2":0.9908878,"cid":23},
|
||||
{"name":"backpack", "confidence":0.963343,"x1":0.1273736,"y1":0.57658505,"x2":0.47765,"y2":0.6986431,"cid":23}
|
||||
]
|
||||
}
|
||||
....
|
||||
|
||||
The `detected_objects` header format is:
|
||||
|
||||
* *object-name*:**confidence** - human readable name of the detected object (e.g. label) with its confidence as a float between [0-1]
|
||||
* *x1*, *y1*, *x2*, *y2* - Response also provides the bounding box of the detected objects represented as `(x1, y1, x2, y2)`. The coordinates are relative to the size of the image size.
|
||||
* *cid* - Classification identifier as defined in the provided https://github.com/tensorflow/models/tree/865c14c/research/object_detection/data[labels] configuration file.
|
||||
|
||||
== Payload
|
||||
|
||||
The incoming type is `byte[]`, and the content type is `application/octet-stream`. The processor processes the input `byte[]` image and outputs an augmented `byte[]` image payload and a JSON header (`detected_objects`).
|
||||
|
||||
== Options
|
||||
|
||||
//tag::configuration-properties[]
|
||||
$$object.detection.cache-model$$:: $$<documentation missing>$$ *($$Boolean$$, default: `$$<none>$$`)*
|
||||
$$object.detection.confidence$$:: $$<documentation missing>$$ *($$Float$$, default: `$$<none>$$`)*
|
||||
$$object.detection.debug-output$$:: $$<documentation missing>$$ *($$Boolean$$, default: `$$<none>$$`)*
|
||||
$$object.detection.debug-output-path$$:: $$<documentation missing>$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$object.detection.labels$$:: $$Labels URI.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$object.detection.model$$:: $$pre-trained tensorflow object detection model.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$object.detection.response-size$$:: $$<documentation missing>$$ *($$Integer$$, default: `$$<none>$$`)*
|
||||
$$object.detection.with-masks$$:: $$<documentation missing>$$ *($$Boolean$$, default: `$$<none>$$`)*
|
||||
//end::configuration-properties[]
|
||||
|
||||
//end::ref-doc[]
|
||||
@@ -1,68 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>object-detection-processor</artifactId>
|
||||
<name>object-detection-processor</name>
|
||||
<description>Object Detection (tensorflow) processor apps</description>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<parent>
|
||||
<groupId>org.springframework.cloud.stream.app</groupId>
|
||||
<artifactId>stream-applications-core</artifactId>
|
||||
<version>5.0.0-SNAPSHOT</version>
|
||||
<relativePath>../../stream-applications-core/pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-configuration-processor</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>object-detection-function</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-deploy-plugin</artifactId>
|
||||
<version>3.0.0</version>
|
||||
<configuration>
|
||||
<skip>false</skip>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-dataflow-apps-docs-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-dataflow-apps-generator-plugin</artifactId>
|
||||
<configuration>
|
||||
<application>
|
||||
<name>object-detection</name>
|
||||
<type>processor</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.stream.app.processor.object.detection.ObjectDetectionProcessorConfiguration.class</configClass>
|
||||
<functionDefinition>objectDetection</functionDefinition>
|
||||
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.stream.app</groupId>
|
||||
<artifactId>object-detection-processor</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
</application>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
@@ -1,100 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020-2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.app.processor.object.detection;
|
||||
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.function.Function;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.fn.common.tensorflow.deprecated.JsonMapperFunction;
|
||||
import org.springframework.cloud.fn.object.detection.ObjectDetectionImageAugmenter;
|
||||
import org.springframework.cloud.fn.object.detection.ObjectDetectionService;
|
||||
import org.springframework.cloud.fn.object.detection.domain.ObjectDetection;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.integration.support.MessageBuilder;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
*/
|
||||
@Configuration
|
||||
@EnableConfigurationProperties(ObjectDetectionProcessorProperties.class)
|
||||
public class ObjectDetectionProcessorConfiguration {
|
||||
|
||||
private static final Log logger = LogFactory.getLog(ObjectDetectionProcessorConfiguration.class);
|
||||
|
||||
/**
|
||||
* Name of the Message header containing the JSON encoded detected objects.
|
||||
*/
|
||||
public static final String DETECTED_OBJECTS_HEADER = "detected_objects";
|
||||
|
||||
@Bean
|
||||
public ObjectDetectionService objectDetectionService(ObjectDetectionProcessorProperties properties) {
|
||||
return new ObjectDetectionService(properties.getModel(),
|
||||
properties.getLabels(), properties.getConfidence(), properties.isWithMasks(),
|
||||
properties.isCacheModel());
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Function<Message<byte[]>, Message<byte[]>> objectDetection(
|
||||
ObjectDetectionService objectDetectionService,
|
||||
ObjectDetectionProcessorProperties properties) {
|
||||
|
||||
return input -> {
|
||||
// You can use file:, http: or classpath: to provide the path to the input image.
|
||||
byte[] inputImage = input.getPayload();
|
||||
|
||||
List<ObjectDetection> detectedObjects = objectDetectionService.detect(inputImage);
|
||||
|
||||
if (!CollectionUtils.isEmpty(detectedObjects) && properties.getResponseSize() < detectedObjects.size()) {
|
||||
detectedObjects = detectedObjects.subList(0, properties.getResponseSize());
|
||||
}
|
||||
|
||||
// Draw the predicted labels on top of the input image.
|
||||
byte[] augmentedImage = new ObjectDetectionImageAugmenter().apply(inputImage, detectedObjects);
|
||||
|
||||
String jsonDetectedObjects = new JsonMapperFunction().apply(detectedObjects);
|
||||
|
||||
Message<byte[]> outMessage = MessageBuilder
|
||||
.withPayload(augmentedImage)
|
||||
.setHeader(DETECTED_OBJECTS_HEADER, jsonDetectedObjects)
|
||||
.build();
|
||||
|
||||
if (properties.isDebugOutput()) {
|
||||
try {
|
||||
logger.info("detected objects = " + jsonDetectedObjects);
|
||||
IOUtils.write(augmentedImage, new FileOutputStream(properties.getDebugOutputPath()));
|
||||
}
|
||||
catch (IOException e) {
|
||||
logger.warn("Cloud not produce debug output", e);
|
||||
}
|
||||
}
|
||||
|
||||
return outMessage;
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,114 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020-2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.app.processor.object.detection;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
*/
|
||||
@ConfigurationProperties("object.detection")
|
||||
@Validated
|
||||
public class ObjectDetectionProcessorProperties {
|
||||
|
||||
/**
|
||||
* pre-trained tensorflow object detection model.
|
||||
*/
|
||||
private String model = "https://download.tensorflow.org/models/object_detection/ssdlite_mobilenet_v2_coco_2018_05_09.tar.gz#frozen_inference_graph.pb";
|
||||
|
||||
/**
|
||||
* Labels URI.
|
||||
*/
|
||||
private String labels = "https://storage.googleapis.com/scdf-tensorflow-models/object-detection/mscoco_label_map.pbtxt";
|
||||
|
||||
private float confidence = 0.4f;
|
||||
|
||||
private boolean withMasks;
|
||||
|
||||
private boolean cacheModel = true;
|
||||
|
||||
private boolean debugOutput = false;
|
||||
|
||||
private String debugOutputPath = "object-detection-result.png";
|
||||
|
||||
private int responseSize = Integer.MAX_VALUE;
|
||||
|
||||
public boolean isDebugOutput() {
|
||||
return debugOutput;
|
||||
}
|
||||
|
||||
public void setDebugOutput(boolean debugOutput) {
|
||||
this.debugOutput = debugOutput;
|
||||
}
|
||||
|
||||
public String getDebugOutputPath() {
|
||||
return debugOutputPath;
|
||||
}
|
||||
|
||||
public void setDebugOutputPath(String debugOutputPath) {
|
||||
this.debugOutputPath = debugOutputPath;
|
||||
}
|
||||
|
||||
public String getModel() {
|
||||
return model;
|
||||
}
|
||||
|
||||
public void setModel(String model) {
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
public String getLabels() {
|
||||
return labels;
|
||||
}
|
||||
|
||||
public void setLabels(String labels) {
|
||||
this.labels = labels;
|
||||
}
|
||||
|
||||
public float getConfidence() {
|
||||
return confidence;
|
||||
}
|
||||
|
||||
public void setConfidence(float confidence) {
|
||||
this.confidence = confidence;
|
||||
}
|
||||
|
||||
public boolean isWithMasks() {
|
||||
return withMasks;
|
||||
}
|
||||
|
||||
public void setWithMasks(boolean withMasks) {
|
||||
this.withMasks = withMasks;
|
||||
}
|
||||
|
||||
public boolean isCacheModel() {
|
||||
return cacheModel;
|
||||
}
|
||||
|
||||
public void setCacheModel(boolean cacheModel) {
|
||||
this.cacheModel = cacheModel;
|
||||
}
|
||||
|
||||
public int getResponseSize() {
|
||||
return responseSize;
|
||||
}
|
||||
|
||||
public void setResponseSize(int responseSize) {
|
||||
this.responseSize = responseSize;
|
||||
}
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
configuration-properties.classes=\
|
||||
org.springframework.cloud.stream.app.processor.object.detection.ObjectDetectionProcessorProperties
|
||||
@@ -1,100 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020-2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.app.processor.object.detection;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import org.springframework.boot.WebApplicationType;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
import org.springframework.cloud.fn.common.tensorflow.deprecated.GraphicsUtils;
|
||||
import org.springframework.cloud.stream.binder.test.InputDestination;
|
||||
import org.springframework.cloud.stream.binder.test.OutputDestination;
|
||||
import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.messaging.support.GenericMessage;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
*/
|
||||
public class ObjectDetectionProcessorTests {
|
||||
|
||||
@Test
|
||||
public void testObjectDetectionProcessor() throws IOException {
|
||||
try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
|
||||
TestChannelBinderConfiguration.getCompleteConfiguration(ObjectDetectionProcessorTestApplication.class))
|
||||
.web(WebApplicationType.NONE)
|
||||
.run("--spring.cloud.function.definition=objectDetection",
|
||||
"--object.detection.model=https://download.tensorflow.org/models/object_detection/ssdlite_mobilenet_v2_coco_2018_05_09.tar.gz#frozen_inference_graph.pb",
|
||||
"--object.detection.labels=https://storage.googleapis.com/scdf-tensorflow-models/object-detection/mscoco_label_map.pbtxt",
|
||||
"--object.detection.responseSize=10",
|
||||
"--object.detection.debugOutput=true",
|
||||
"--object.detection.debugOutputPath=./target/object-detection-1.png")) {
|
||||
|
||||
InputDestination processorInput = context.getBean(InputDestination.class);
|
||||
OutputDestination processorOutput = context.getBean(OutputDestination.class);
|
||||
|
||||
byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/pivotal.jpeg");
|
||||
processorInput.send(new GenericMessage<>(inputImage));
|
||||
Message<byte[]> sourceMessage = processorOutput.receive(10000);
|
||||
String jsonRecognizedObjects = (String) sourceMessage.getHeaders().get(ObjectDetectionProcessorConfiguration.DETECTED_OBJECTS_HEADER);
|
||||
assertThat(jsonRecognizedObjects).isNotEmpty();
|
||||
//assertThat(jsonRecognizedObjects)
|
||||
// .isEqualTo("[{\"label\":\"giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca\",\"probability\":0.962329626083374}," +
|
||||
// "{\"label\":\"badger\",\"probability\":0.006058811210095882}," +
|
||||
// "{\"label\":\"ram, tup\",\"probability\":0.0010668420000001788}]");
|
||||
}
|
||||
}
|
||||
|
||||
//@Test
|
||||
public void testObjectDetectionProcessoriNaturalistSpecies() throws IOException {
|
||||
try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
|
||||
TestChannelBinderConfiguration.getCompleteConfiguration(ObjectDetectionProcessorTestApplication.class))
|
||||
.web(WebApplicationType.NONE)
|
||||
.run(
|
||||
//"--spring.cloud.function.definition=objectDetection",
|
||||
"--object.detection.model=https://download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_fgvc_2018_07_19.tar.gz#frozen_inference_graph.pb",
|
||||
"--object.detection.labels=https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/fgvc_2854_classes_label_map.pbtxt",
|
||||
"--object.detection..debugOutput=true",
|
||||
"--object.detection..debugOutputPath=./target/object-detection-2.png")) {
|
||||
|
||||
InputDestination processorInput = context.getBean(InputDestination.class);
|
||||
OutputDestination processorOutput = context.getBean(OutputDestination.class);
|
||||
|
||||
byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/animals2.jpg");
|
||||
processorInput.send(new GenericMessage<>(inputImage));
|
||||
Message<byte[]> sourceMessage = processorOutput.receive(10000);
|
||||
String jsonRecognizedObjects = (String) sourceMessage.getHeaders().get(ObjectDetectionProcessorConfiguration.DETECTED_OBJECTS_HEADER);
|
||||
//assertThat(jsonRecognizedObjects)
|
||||
// .isEqualTo("[{\"label\":\"giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca\",\"probability\":0.962329626083374}," +
|
||||
// "{\"label\":\"badger\",\"probability\":0.006058811210095882}," +
|
||||
// "{\"label\":\"ram, tup\",\"probability\":0.0010668420000001788}]");
|
||||
}
|
||||
}
|
||||
|
||||
@SpringBootApplication
|
||||
@Import({ ObjectDetectionProcessorConfiguration.class })
|
||||
public static class ObjectDetectionProcessorTestApplication {
|
||||
}
|
||||
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 177 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.4 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 11 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 125 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 159 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 292 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 52 KiB |
@@ -21,9 +21,6 @@
|
||||
<module>transform-processor</module>
|
||||
<module>script-processor</module>
|
||||
<module>twitter-trend-processor</module>
|
||||
<module>image-recognition-processor</module>
|
||||
<module>object-detection-processor</module>
|
||||
<module>semantic-segmentation-processor</module>
|
||||
</modules>
|
||||
<build>
|
||||
<plugins>
|
||||
|
||||
@@ -78,8 +78,8 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>payload-converter-function</artifactId>
|
||||
<version>${java-functions.version}</version>
|
||||
<artifactId>spring-payload-converter-function</artifactId>
|
||||
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2016-2020 the original author or authors.
|
||||
* Copyright 2016-2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -30,6 +30,7 @@ import org.springframework.core.io.ByteArrayResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.integration.handler.MessageProcessor;
|
||||
import org.springframework.integration.scripting.ScriptVariableGenerator;
|
||||
import org.springframework.integration.scripting.dsl.ScriptSpec;
|
||||
import org.springframework.integration.scripting.dsl.Scripts;
|
||||
import org.springframework.messaging.Message;
|
||||
|
||||
@@ -47,7 +48,7 @@ import org.springframework.messaging.Message;
|
||||
* @author Artme Bilan
|
||||
* @author Soby Chacko
|
||||
*/
|
||||
@Configuration
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
@EnableConfigurationProperties(ScriptProcessorProperties.class)
|
||||
@Import(ScriptVariableGeneratorConfiguration.class)
|
||||
public class ScriptProcessorConfiguration {
|
||||
@@ -69,12 +70,12 @@ public class ScriptProcessorConfiguration {
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Function<Message<?>, Object> scriptProcessorFunction() {
|
||||
return processor()::processMessage;
|
||||
public Function<Message<?>, Object> scriptProcessorFunction(MessageProcessor<?> messageProcessor) {
|
||||
return messageProcessor::processMessage;
|
||||
}
|
||||
|
||||
@Bean
|
||||
public MessageProcessor<?> processor() {
|
||||
public ScriptSpec processor() {
|
||||
String language = this.properties.getLanguage();
|
||||
String script = this.properties.getScript();
|
||||
logger.info(String.format("Input script is '%s', language is '%s'", script, language));
|
||||
@@ -82,8 +83,7 @@ public class ScriptProcessorConfiguration {
|
||||
|
||||
return Scripts.processor(scriptResource)
|
||||
.lang(language)
|
||||
.variableGenerator(scriptVariableGenerator)
|
||||
.get();
|
||||
.variableGenerator(scriptVariableGenerator);
|
||||
}
|
||||
|
||||
private static String decodeScript(String script) {
|
||||
@@ -94,4 +94,5 @@ public class ScriptProcessorConfiguration {
|
||||
}
|
||||
return toProcess.replaceAll(NEWLINE_ESCAPE, "\n").replaceAll(DOUBLE_DOUBLE_QUOTE, "\"");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
//tag::ref-doc[]
|
||||
|
||||
= Semantic Segmentation Processor
|
||||
|
||||
Image Semantic Segmentation based on the state-of-art https://github.com/tensorflow/models/tree/master/research/deeplab[DeepLab] Tensorflow model.
|
||||
|
||||
The `Semantic Segmentation` is the process of associating each pixel of an image with a class label, (such as flower, person, road, sky, ocean, or car).
|
||||
Unlike the `Instance Segmentation`, which produces instance-aware region masks, the `Semantic Segmentation` produces class-aware masks.
|
||||
For implementing `Instance Segmentation` consult the https://github.com/spring-cloud/stream-applications/tree/master/functions/function/object-detection-function[Object Detection Service] instead.
|
||||
|
||||
The `Semantic Segmentation Processor` uses the https://github.com/spring-cloud/stream-applications/tree/master/functions/function/semantic-segmentation-function[Semantic Segmentation Function] library and the https://github.com/spring-cloud/stream-applications/tree/master/functions/common/tensorflow-common[TensorFlow Service].
|
||||
|
||||
== Payload
|
||||
|
||||
The incoming type is `byte[]`, and the content type is `application/octet-stream`. The processor processes the input `byte[]` image and outputs augmented `byte[]` image payload and json header.
|
||||
|
||||
Processor's input is an image byte array, and the output is an augmented image byte array, and a JSON header `semantic_segmentation` in this format:
|
||||
|
||||
[source,json]
|
||||
....
|
||||
[
|
||||
[ 0, 0, 0 ],
|
||||
[ 127, 127, 127 ],
|
||||
[ 255, 255, 255 ],
|
||||
...
|
||||
]
|
||||
....
|
||||
|
||||
The output header json format represents the color pixel map computed from the input image.
|
||||
|
||||
== Options
|
||||
|
||||
//tag::configuration-properties[]
|
||||
$$semantic.segmentation.color-map-uri$$:: $$Every pre-trained model is based on certain object color maps. The pre-defined options are: - classpath:/colormap/citymap_colormap.json - classpath:/colormap/ade20k_colormap.json - classpath:/colormap/black_white_colormap.json - classpath:/colormap/mapillary_colormap.json$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$semantic.segmentation.debug-output$$:: $$save output image inn the local debugOutputPath path.$$ *($$Boolean$$, default: `$$<none>$$`)*
|
||||
$$semantic.segmentation.debug-output-path$$:: $$<documentation missing>$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$semantic.segmentation.mask-transparency$$:: $$The alpha color of the computed segmentation mask image.$$ *($$Float$$, default: `$$<none>$$`)*
|
||||
$$semantic.segmentation.model$$:: $$pre-trained tensorflow semantic segmentation model.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$semantic.segmentation.output-type$$:: $$Specifies the output image type. You can return either the input image with the computed mask overlay, or the mask alone.$$ *($$OutputType$$, default: `$$<none>$$`, possible values: `blended`,`mask`)*
|
||||
//end::configuration-properties[]
|
||||
|
||||
//end::ref-doc[]
|
||||
@@ -1,73 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>semantic-segmentation-processor</artifactId>
|
||||
<name>semantic-segmentation-processor</name>
|
||||
<description>Semantic Segmentation (tensorflow) processor apps</description>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<parent>
|
||||
<groupId>org.springframework.cloud.stream.app</groupId>
|
||||
<artifactId>stream-applications-core</artifactId>
|
||||
<version>5.0.0-SNAPSHOT</version>
|
||||
<relativePath>../../stream-applications-core/pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-configuration-processor</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>semantic-segmentation-function</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>object-detection-function</artifactId>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-deploy-plugin</artifactId>
|
||||
<version>3.0.0</version>
|
||||
<configuration>
|
||||
<skip>false</skip>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-dataflow-apps-docs-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-dataflow-apps-generator-plugin</artifactId>
|
||||
<configuration>
|
||||
<application>
|
||||
<name>semantic-segmentation</name>
|
||||
<type>processor</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.stream.app.processor.semantic.segmentation.SemanticSegmentationProcessorConfiguration.class</configClass>
|
||||
<functionDefinition>semanticSegmentationFunction</functionDefinition>
|
||||
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.stream.app</groupId>
|
||||
<artifactId>semantic-segmentation-processor</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
</application>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
@@ -1,89 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020-2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.app.processor.semantic.segmentation;
|
||||
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.function.Function;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.fn.common.tensorflow.deprecated.JsonMapperFunction;
|
||||
import org.springframework.cloud.fn.semantic.segmentation.SegmentationColorMap;
|
||||
import org.springframework.cloud.fn.semantic.segmentation.SemanticSegmentation;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.integration.support.MessageBuilder;
|
||||
import org.springframework.messaging.Message;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
*/
|
||||
@Configuration
|
||||
@EnableConfigurationProperties(SemanticSegmentationProcessorProperties.class)
|
||||
public class SemanticSegmentationProcessorConfiguration {
|
||||
|
||||
private static final Log logger = LogFactory.getLog(SemanticSegmentationProcessorConfiguration.class);
|
||||
|
||||
/**
|
||||
* Output header name.
|
||||
*/
|
||||
public static final String SEMANTIC_SEGMENTATION_HEADER = "semantic_segmentation";
|
||||
|
||||
@Bean
|
||||
public SemanticSegmentation semanticSegmentation(SemanticSegmentationProcessorProperties properties) {
|
||||
return new SemanticSegmentation(properties.getModel(),
|
||||
SegmentationColorMap.loadColorMap(properties.getColorMapUri()), null,
|
||||
properties.getMaskTransparency());
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Function<Message<byte[]>, Message<byte[]>> semanticSegmentationFunction(
|
||||
SemanticSegmentation semanticSegmentation,
|
||||
SemanticSegmentationProcessorProperties properties) {
|
||||
|
||||
return input -> {
|
||||
// You can use file:, http: or classpath: to provide the path to the input image.
|
||||
byte[] inputImage = input.getPayload();
|
||||
|
||||
byte[] outputImage = (properties.getOutputType() == SemanticSegmentationProcessorProperties.OutputType.blended) ?
|
||||
semanticSegmentation.blendMask(inputImage) : semanticSegmentation.maskImage(inputImage);
|
||||
|
||||
long[][] maskPixels = semanticSegmentation.maskPixels(inputImage);
|
||||
String jsonMaskPixels = new JsonMapperFunction().apply(maskPixels);
|
||||
|
||||
Message<byte[]> outMessage = MessageBuilder
|
||||
.withPayload(outputImage)
|
||||
.setHeader(SEMANTIC_SEGMENTATION_HEADER, jsonMaskPixels)
|
||||
.build();
|
||||
|
||||
if (properties.isDebugOutput()) {
|
||||
try {
|
||||
IOUtils.write(outputImage, new FileOutputStream(properties.getDebugOutputPath()));
|
||||
}
|
||||
catch (IOException e) {
|
||||
logger.warn("Cloud not produce debug output", e);
|
||||
}
|
||||
}
|
||||
|
||||
return outMessage;
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020-2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.app.processor.semantic.segmentation;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
*/
|
||||
@ConfigurationProperties("semantic.segmentation")
|
||||
@Validated
|
||||
public class SemanticSegmentationProcessorProperties {
|
||||
|
||||
enum OutputType {
|
||||
/** Input image augmented with the segmentation mask on top. */
|
||||
blended,
|
||||
/** Image of the segmentation mask. */
|
||||
mask
|
||||
}
|
||||
|
||||
/**
|
||||
* pre-trained tensorflow semantic segmentation model.
|
||||
*/
|
||||
private String model = "https://download.tensorflow.org/models/deeplabv3_mnv2_cityscapes_train_2018_02_05.tar.gz#frozen_inference_graph.pb";
|
||||
|
||||
/**
|
||||
* Specifies the output image type. You can return either the input image with the computed mask overlay, or
|
||||
* the mask alone.
|
||||
*/
|
||||
private OutputType outputType = OutputType.blended;
|
||||
|
||||
/**
|
||||
* Every pre-trained model is based on certain object color maps.
|
||||
* The pre-defined options are:
|
||||
* - classpath:/colormap/citymap_colormap.json
|
||||
* - classpath:/colormap/ade20k_colormap.json
|
||||
* - classpath:/colormap/black_white_colormap.json
|
||||
* - classpath:/colormap/mapillary_colormap.json
|
||||
*/
|
||||
private String colorMapUri = "classpath:/colormap/citymap_colormap.json";
|
||||
|
||||
/**
|
||||
* The alpha color of the computed segmentation mask image.
|
||||
*/
|
||||
private float maskTransparency = 0.45f;
|
||||
|
||||
/**
|
||||
* save output image inn the local debugOutputPath path.
|
||||
*/
|
||||
private boolean debugOutput = false;
|
||||
|
||||
private String debugOutputPath = "semantic-segmentation-result.png";
|
||||
|
||||
public OutputType getOutputType() {
|
||||
return outputType;
|
||||
}
|
||||
|
||||
public void setOutputType(OutputType outputType) {
|
||||
this.outputType = outputType;
|
||||
}
|
||||
|
||||
public boolean isDebugOutput() {
|
||||
return debugOutput;
|
||||
}
|
||||
|
||||
public void setDebugOutput(boolean debugOutput) {
|
||||
this.debugOutput = debugOutput;
|
||||
}
|
||||
|
||||
public String getDebugOutputPath() {
|
||||
return debugOutputPath;
|
||||
}
|
||||
|
||||
public void setDebugOutputPath(String debugOutputPath) {
|
||||
this.debugOutputPath = debugOutputPath;
|
||||
}
|
||||
|
||||
public String getModel() {
|
||||
return model;
|
||||
}
|
||||
|
||||
public void setModel(String model) {
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
public String getColorMapUri() {
|
||||
return colorMapUri;
|
||||
}
|
||||
|
||||
public void setColorMapUri(String colorMapUri) {
|
||||
this.colorMapUri = colorMapUri;
|
||||
}
|
||||
|
||||
public float getMaskTransparency() {
|
||||
return maskTransparency;
|
||||
}
|
||||
|
||||
public void setMaskTransparency(float maskTransparency) {
|
||||
this.maskTransparency = maskTransparency;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
configuration-properties.classes=\
|
||||
org.springframework.cloud.stream.app.processor.semantic.segmentation.SemanticSegmentationProcessorProperties
|
||||
@@ -1,76 +0,0 @@
|
||||
/*
|
||||
* Copyright 2020-2020 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.cloud.stream.app.processor.semantic.segmentation;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import org.springframework.boot.WebApplicationType;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
import org.springframework.cloud.fn.common.tensorflow.deprecated.GraphicsUtils;
|
||||
import org.springframework.cloud.stream.binder.test.InputDestination;
|
||||
import org.springframework.cloud.stream.binder.test.OutputDestination;
|
||||
import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.messaging.support.GenericMessage;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
*/
|
||||
public class SemanticSegmentationProcessorTests {
|
||||
|
||||
@Test
|
||||
public void testSemanticSegmentationProcessor() throws IOException {
|
||||
try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
|
||||
TestChannelBinderConfiguration.getCompleteConfiguration(SemanticSegmentationTestApplication.class))
|
||||
.web(WebApplicationType.NONE)
|
||||
.run("--spring.cloud.function.definition=semanticSegmentationFunction",
|
||||
"--semantic.segmentation.model=https://download.tensorflow.org/models/deeplabv3_mnv2_cityscapes_train_2018_02_05.tar.gz#frozen_inference_graph.pb",
|
||||
"--semantic.segmentation.colorMapUri=classpath:/colormap/citymap_colormap.json",
|
||||
"--semantic.segmentation.outputType=blended",
|
||||
"--semantic.segmentation.debugOutput=true",
|
||||
"--semantic.segmentation.debugOutputPath=./target/semantic-segmentation-1.png")) {
|
||||
|
||||
InputDestination processorInput = context.getBean(InputDestination.class);
|
||||
OutputDestination processorOutput = context.getBean(OutputDestination.class);
|
||||
|
||||
byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/amsterdam-cityscape1.jpg");
|
||||
processorInput.send(new GenericMessage<>(inputImage));
|
||||
Message<byte[]> sourceMessage = processorOutput.receive(10000);
|
||||
String jsonRecognizedObjects = (String) sourceMessage.getHeaders().get(
|
||||
SemanticSegmentationProcessorConfiguration.SEMANTIC_SEGMENTATION_HEADER);
|
||||
assertThat(jsonRecognizedObjects).isNotEmpty();
|
||||
//assertThat(jsonRecognizedObjects)
|
||||
// .isEqualTo("[{\"label\":\"giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca\",\"probability\":0.962329626083374}," +
|
||||
// "{\"label\":\"badger\",\"probability\":0.006058811210095882}," +
|
||||
// "{\"label\":\"ram, tup\",\"probability\":0.0010668420000001788}]");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@SpringBootApplication
|
||||
@Import(SemanticSegmentationProcessorConfiguration.class)
|
||||
public static class SemanticSegmentationTestApplication {
|
||||
}
|
||||
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 102 KiB |
@@ -20,7 +20,7 @@ If the incoming type is `byte[]` and the content type is set to `text/plain` or
|
||||
$$splitter.apply-sequence$$:: $$Add correlation/sequence information in headers to facilitate later aggregation.$$ *($$Boolean$$, default: `$$true$$`)*
|
||||
$$splitter.charset$$:: $$The charset to use when converting bytes in text-based files to String.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$splitter.delimiters$$:: $$When expression is null, delimiters to use when tokenizing {@link String} payloads.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$splitter.expression$$:: $$A SpEL expression for splitting payloads.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$splitter.expression$$:: $$A SpEL expression for splitting payloads.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$splitter.file-markers$$:: $$Set to true or false to use a {@code FileSplitter} (to split text-based files by line) that includes (or not) beginning/end of file markers.$$ *($$Boolean$$, default: `$$<none>$$`)*
|
||||
$$splitter.markers-json$$:: $$When 'fileMarkers == true', specify if they should be produced as FileSplitter.FileMarker objects or JSON.$$ *($$Boolean$$, default: `$$true$$`)*
|
||||
//end::configuration-properties[]
|
||||
|
||||
@@ -17,11 +17,11 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>splitter-function</artifactId>
|
||||
<artifactId>spring-splitter-function</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>payload-converter-function</artifactId>
|
||||
<artifactId>spring-payload-converter-function</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
@@ -47,19 +47,18 @@
|
||||
<name>splitter</name>
|
||||
<type>processor</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.splitter.SplitterFunctionConfiguration.class
|
||||
</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>byteArrayTextToString|splitterFunction</functionDefinition>
|
||||
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>payload-converter-function</artifactId>
|
||||
<artifactId>spring-payload-converter-function</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>splitter-function</artifactId>
|
||||
<artifactId>spring-splitter-function</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -21,7 +21,7 @@ The incoming message can contain any type of payload.
|
||||
== Options
|
||||
|
||||
//tag::configuration-properties[]
|
||||
$$spel.function.expression$$:: $$A SpEL expression to apply.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$spel.function.expression$$:: $$A SpEL expression to apply.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
//end::configuration-properties[]
|
||||
|
||||
//end::ref-doc[]
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>spel-function</artifactId>
|
||||
<artifactId>spring-spel-function</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
@@ -43,13 +43,13 @@
|
||||
<name>transform</name>
|
||||
<type>processor</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.spel.SpelFunctionConfiguration.class</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>byteArrayTextToString|spelFunction</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>spel-function</artifactId>
|
||||
<artifactId>spring-spel-function</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -31,12 +31,12 @@ Properties grouped by prefix:
|
||||
|
||||
=== twitter.trend.closest
|
||||
|
||||
$$lat$$:: $$If provided with a long parameter the available trend locations will be sorted by distance, nearest to furthest, to the co-ordinate pair. The valid ranges for longitude is -180.0 to +180.0 (West is negative, East is positive) inclusive.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$lon$$:: $$If provided with a lat parameter the available trend locations will be sorted by distance, nearest to furthest, to the co-ordinate pair. The valid ranges for longitude is -180.0 to +180.0 (West is negative, East is positive) inclusive.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$lat$$:: $$If provided with a long parameter the available trend locations will be sorted by distance, nearest to the furthest, to the co-ordinate pair. The valid ranges for longitude is -180.0 to +180.0 (West is negative, East is positive) inclusive.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$lon$$:: $$If provided with a lat parameter the available trend locations will be sorted by distance, nearest to the furthest, to the co-ordinate pair. The valid ranges for longitude is -180.0 to +180.0 (West is negative, East is positive) inclusive.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
|
||||
=== twitter.trend
|
||||
|
||||
$$location-id$$:: $$The Yahoo! Where On Earth ID of the location to return trending information for. Global information is available by using 1 as the WOEID.$$ *($$Expression$$, default: `$$payload$$`)*
|
||||
$$location-id$$:: $$The Yahoo! Where On Earth ID of the location to return trending information for. Global information is available by using 1 as the WOEID.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$trend-query-type$$:: $$<documentation missing>$$ *($$TrendQueryType$$, default: `$$<none>$$`, possible values: `trend`,`trendLocation`)*
|
||||
//end::configuration-properties[]
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>twitter-function</artifactId>
|
||||
<artifactId>spring-twitter-function</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
@@ -28,13 +28,6 @@
|
||||
<dependency>
|
||||
<groupId>org.mock-server</groupId>
|
||||
<artifactId>mockserver-netty</artifactId>
|
||||
<version>${mockserver.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mock-server</groupId>
|
||||
<artifactId>mockserver-client-java</artifactId>
|
||||
<version>${mockserver.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
@@ -61,14 +54,14 @@
|
||||
<name>twitter-trend</name>
|
||||
<type>processor</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.twitter.trend.TwitterTrendFunctionConfiguration.class</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>twitterTrendFunction</functionDefinition>
|
||||
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>twitter-function</artifactId>
|
||||
<artifactId>spring-twitter-function</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -18,7 +18,6 @@ package org.springframework.cloud.stream.app.processor.twitter.trend;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Function;
|
||||
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
@@ -36,17 +35,14 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
import org.springframework.cloud.fn.common.twitter.TwitterConnectionProperties;
|
||||
import org.springframework.cloud.fn.common.twitter.util.TwitterTestUtils;
|
||||
import org.springframework.cloud.fn.twitter.trend.TwitterTrendFunctionConfiguration;
|
||||
import org.springframework.cloud.stream.binder.test.InputDestination;
|
||||
import org.springframework.cloud.stream.binder.test.OutputDestination;
|
||||
import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.messaging.support.GenericMessage;
|
||||
import org.springframework.test.util.TestSocketUtils;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockserver.matchers.Times.exactly;
|
||||
@@ -56,22 +52,20 @@ import static org.mockserver.verify.VerificationTimes.once;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
* @author Artem Bilan
|
||||
*/
|
||||
public class TwitterTrendProcessorIntegrationTests {
|
||||
|
||||
private static final String MOCK_SERVER_IP = "127.0.0.1";
|
||||
|
||||
private static final Integer MOCK_SERVER_PORT = TestSocketUtils.findAvailableTcpPort();
|
||||
|
||||
private static ClientAndServer mockServer;
|
||||
|
||||
private static MockServerClient mockClient;
|
||||
|
||||
private static HttpRequest trendsRequest;
|
||||
|
||||
@BeforeAll
|
||||
public static void startServer() {
|
||||
mockServer = ClientAndServer.startClientAndServer(MOCK_SERVER_PORT);
|
||||
mockClient = new MockServerClient(MOCK_SERVER_IP, MOCK_SERVER_PORT);
|
||||
mockServer = ClientAndServer.startClientAndServer();
|
||||
mockClient = new MockServerClient("localhost", mockServer.getPort());
|
||||
|
||||
trendsRequest = setExpectation(request()
|
||||
.withMethod("GET")
|
||||
@@ -91,6 +85,7 @@ public class TwitterTrendProcessorIntegrationTests {
|
||||
.web(WebApplicationType.NONE)
|
||||
.run("--spring.cloud.function.definition=twitterTrendFunction",
|
||||
|
||||
"--twitter.trend.trend-query-type=trend",
|
||||
"--twitter.trend.locationId='2972'",
|
||||
"--twitter.connection.rawJson=true",
|
||||
|
||||
@@ -110,11 +105,6 @@ public class TwitterTrendProcessorIntegrationTests {
|
||||
assertThat(outputMessage).isNotNull();
|
||||
|
||||
mockClient.verify(trendsRequest, once());
|
||||
|
||||
//Resource trendsResource = new DefaultResourceLoader().getResource("classpath:/response/trends.json");
|
||||
//String expected = new String(StreamUtils.copyToByteArray(trendsResource.getInputStream()), StandardCharsets.UTF_8).trim();
|
||||
//String actual = new String(outputMessage.getPayload(), StandardCharsets.UTF_8);
|
||||
//JSONAssert.assertEquals(expected, actual, JSONCompareMode.LENIENT);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,15 +117,14 @@ public class TwitterTrendProcessorIntegrationTests {
|
||||
new Header("Content-Type", "application/json; charset=utf-8"),
|
||||
new Header("Cache-Control", "public, max-age=86400"))
|
||||
.withBody(TwitterTestUtils.asString("classpath:/response/trends.json"))
|
||||
.withDelay(TimeUnit.SECONDS, 1)
|
||||
);
|
||||
return request;
|
||||
}
|
||||
|
||||
@SpringBootConfiguration
|
||||
@EnableAutoConfiguration
|
||||
@Import(TwitterTrendFunctionConfiguration.class)
|
||||
public static class TestTwitterTrendProcessorApplication {
|
||||
|
||||
@Bean
|
||||
@Primary
|
||||
public twitter4j.conf.Configuration twitterConfiguration2(TwitterConnectionProperties properties,
|
||||
@@ -143,11 +132,11 @@ public class TwitterTrendProcessorIntegrationTests {
|
||||
|
||||
Function<TwitterConnectionProperties, ConfigurationBuilder> mockedConfiguration =
|
||||
toConfigurationBuilder.andThen(
|
||||
new TwitterTestUtils().mockTwitterUrls(
|
||||
String.format("http://%s:%s", MOCK_SERVER_IP, MOCK_SERVER_PORT)));
|
||||
new TwitterTestUtils().mockTwitterUrls("http://localhost:" + mockServer.getPort()));
|
||||
|
||||
return mockedConfiguration.apply(properties).build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>analytics-consumer</artifactId>
|
||||
<artifactId>spring-analytics-consumer</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.awaitility</groupId>
|
||||
@@ -55,13 +55,13 @@
|
||||
<name>analytics</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.analytics.AnalyticsConsumerConfiguration.class</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>byteArrayTextToString|analyticsConsumer</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>analytics-consumer</artifactId>
|
||||
<artifactId>spring-analytics-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -24,7 +24,7 @@ $$entity-base-packages$$:: $$Base packages to scan for entities annotated with T
|
||||
$$init-script$$:: $$Resource with CQL scripts (delimited by ';') to initialize keyspace schema.$$ *($$Resource$$, default: `$$<none>$$`)*
|
||||
$$skip-ssl-validation$$:: $$Flag to validate the Servers' SSL certs.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
|
||||
=== cassandra
|
||||
=== cassandra.consumer
|
||||
|
||||
$$consistency-level$$:: $$The consistency level for write operation.$$ *($$ConsistencyLevel$$, default: `$$<none>$$`)*
|
||||
$$ingest-query$$:: $$Ingest Cassandra query.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>cassandra-consumer</artifactId>
|
||||
<artifactId>spring-cassandra-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
@@ -43,15 +43,13 @@
|
||||
<name>cassandra</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>
|
||||
org.springframework.cloud.fn.consumer.cassandra.CassandraConsumerConfiguration.class
|
||||
</configClass>
|
||||
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>cassandraConsumer</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>cassandra-consumer</artifactId>
|
||||
<artifactId>spring-cassandra-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -20,11 +20,11 @@ Properties grouped by prefix:
|
||||
|
||||
=== elasticsearch.consumer
|
||||
|
||||
$$async$$:: $$Indicates whether the indexing operation is async or not. By default indexing is done synchronously.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$async$$:: $$Indicates whether the indexing operation is async or not. By default, indexing is done synchronously.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$batch-size$$:: $$Number of items to index for each request. It defaults to 1. For values greater than 1 bulk indexing API will be used.$$ *($$Integer$$, default: `$$1$$`)*
|
||||
$$group-timeout$$:: $$Timeout in milliseconds after which message group is flushed when bulk indexing is active. It defaults to -1, meaning no automatic flush of idle message groups occurs.$$ *($$Long$$, default: `$$-1$$`)*
|
||||
$$id$$:: $$The id of the document to index. If set, the INDEX_ID header value overrides this property on a per message basis.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$index$$:: $$Name of the index. If set, the INDEX_NAME header value overrides this property on a per message basis.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$id$$:: $$The id of the document to index. If set, the INDEX_ID header value overrides this property on a per-message basis.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$index$$:: $$Name of the index. If set, the INDEX_NAME header value overrides this property on a per-message basis.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$routing$$:: $$Indicates the shard to route to. If not provided, Elasticsearch will default to a hash of the document id.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$timeout-seconds$$:: $$Timeout for the shard to be available. If not set, it defaults to 1 minute set by the Elasticsearch client.$$ *($$Long$$, default: `$$0$$`)*
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>elasticsearch-consumer</artifactId>
|
||||
<artifactId>spring-elasticsearch-consumer</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.awaitility</groupId>
|
||||
@@ -72,14 +72,14 @@
|
||||
<name>elasticsearch</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.elasticsearch.ElasticsearchConsumerConfiguration.class</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>byteArrayTextToString|elasticsearchConsumer</functionDefinition>
|
||||
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>elasticsearch-consumer</artifactId>
|
||||
<artifactId>spring-elasticsearch-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -15,10 +15,10 @@ The file sink app writes each message it receives to a file.
|
||||
The `file-sink` has the following options:
|
||||
|
||||
//tag::configuration-properties[]
|
||||
$$file.consumer.binary$$:: $$A flag to indicate whether adding a newline after the write should be suppressed.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$file.consumer.binary$$:: $$A flag to indicate whether adding a newline after the write operation should be suppressed.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$file.consumer.charset$$:: $$The charset to use when writing text content.$$ *($$String$$, default: `$$UTF-8$$`)*
|
||||
$$file.consumer.directory$$:: $$The parent directory of the target file.$$ *($$File$$, default: `$$<none>$$`)*
|
||||
$$file.consumer.directory-expression$$:: $$The expression to evaluate for the parent directory of the target file.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$file.consumer.directory-expression$$:: $$The expression to evaluate for the parent directory of the target file.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$file.consumer.mode$$:: $$The FileExistsMode to use if the target file already exists.$$ *($$FileExistsMode$$, default: `$$<none>$$`, possible values: `APPEND`,`APPEND_NO_FLUSH`,`FAIL`,`IGNORE`,`REPLACE`,`REPLACE_IF_MODIFIED`)*
|
||||
$$file.consumer.name$$:: $$The name of the target file.$$ *($$String$$, default: `$$file-consumer$$`)*
|
||||
$$file.consumer.name-expression$$:: $$The expression to evaluate for the name of the target file.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>file-consumer</artifactId>
|
||||
<artifactId>spring-file-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
@@ -43,13 +43,13 @@
|
||||
<name>file</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.file.FileConsumerConfiguration.class</configClass>
|
||||
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>fileConsumer</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>file-consumer</artifactId>
|
||||
<artifactId>spring-file-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -37,14 +37,14 @@ Properties grouped by prefix:
|
||||
|
||||
=== ftp.consumer
|
||||
|
||||
$$auto-create-dir$$:: $$Whether or not to create the remote directory.$$ *($$Boolean$$, default: `$$true$$`)*
|
||||
$$auto-create-dir$$:: $$Whether to create the remote directory.$$ *($$Boolean$$, default: `$$true$$`)*
|
||||
$$filename-expression$$:: $$A SpEL expression to generate the remote file name.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$mode$$:: $$Action to take if the remote file already exists.$$ *($$FileExistsMode$$, default: `$$<none>$$`, possible values: `APPEND`,`APPEND_NO_FLUSH`,`FAIL`,`IGNORE`,`REPLACE`,`REPLACE_IF_MODIFIED`)*
|
||||
$$remote-dir$$:: $$The remote FTP directory.$$ *($$String$$, default: `$$/$$`)*
|
||||
$$remote-file-separator$$:: $$The remote file separator.$$ *($$String$$, default: `$$/$$`)*
|
||||
$$temporary-remote-dir$$:: $$A temporary directory where the file will be written if '#isUseTemporaryFilename()' is true.$$ *($$String$$, default: `$$/$$`)*
|
||||
$$tmp-file-suffix$$:: $$The suffix to use while the transfer is in progress.$$ *($$String$$, default: `$$.tmp$$`)*
|
||||
$$use-temporary-filename$$:: $$Whether or not to write to a temporary file and rename.$$ *($$Boolean$$, default: `$$true$$`)*
|
||||
$$use-temporary-filename$$:: $$Whether to write to a temporary file and rename.$$ *($$Boolean$$, default: `$$true$$`)*
|
||||
|
||||
=== ftp.factory
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>ftp-consumer</artifactId>
|
||||
<artifactId>spring-ftp-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
@@ -43,12 +43,13 @@
|
||||
<name>ftp</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.ftp.FtpConsumerConfiguration.class</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>ftpConsumer</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>ftp-consumer</artifactId>
|
||||
<artifactId>spring-ftp-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>jdbc-consumer</artifactId>
|
||||
<artifactId>spring-jdbc-consumer</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.h2database</groupId>
|
||||
@@ -54,12 +54,13 @@
|
||||
<name>jdbc</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.jdbc.JdbcConsumerConfiguration.class</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>jdbcConsumer</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>jdbc-consumer</artifactId>
|
||||
<artifactId>spring-jdbc-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -51,6 +51,7 @@ $$value-serializer$$:: $$Serializer class for values.$$ *($$Class<?>$$, default:
|
||||
=== spring.kafka.template
|
||||
|
||||
$$default-topic$$:: $$Default topic to which messages are sent.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$observation-enabled$$:: $$Whether to enable observation.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$transaction-id-prefix$$:: $$Transaction id prefix, override the transaction id prefix in the producer factory.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
//end::configuration-properties[]
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>kafka-publisher</artifactId>
|
||||
<artifactId>spring-kafka-publisher</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2023-2023 the original author or authors.
|
||||
* Copyright 2023-2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -46,7 +46,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
"kafka.publisher.topic=" + KafkaSinkTests.TEST_TOPIC,
|
||||
"kafka.publisher.mappedHeaders=mapped"
|
||||
})
|
||||
@EmbeddedKafka(bootstrapServersProperty = "spring.kafka.bootstrap-servers")
|
||||
@EmbeddedKafka(kraft = false)
|
||||
@DirtiesContext
|
||||
public class KafkaSinkTests {
|
||||
|
||||
|
||||
@@ -13,9 +13,9 @@ The **$$log$$** $$sink$$ has the following options:
|
||||
|
||||
|
||||
//tag::configuration-properties[]
|
||||
$$log.expression$$:: $$A SpEL expression (against the incoming message) to evaluate as the logged message.$$ *($$String$$, default: `$$payload$$`)*
|
||||
$$log.level$$:: $$The level at which to log messages.$$ *($$Level$$, default: `$$<none>$$`, possible values: `FATAL`,`ERROR`,`WARN`,`INFO`,`DEBUG`,`TRACE`)*
|
||||
$$log.name$$:: $$The name of the logger to use.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$log.consumer.expression$$:: $$A SpEL expression (against the incoming message) to evaluate as the logged message.$$ *($$String$$, default: `$$payload$$`)*
|
||||
$$log.consumer.level$$:: $$The level at which to log messages.$$ *($$Level$$, default: `$$<none>$$`, possible values: `FATAL`,`ERROR`,`WARN`,`INFO`,`DEBUG`,`TRACE`)*
|
||||
$$log.consumer.name$$:: $$The name of the logger to use.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
//end::configuration-properties[]
|
||||
|
||||
//end::ref-doc[]
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>log-consumer</artifactId>
|
||||
<artifactId>spring-log-consumer</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.awaitility</groupId>
|
||||
@@ -54,14 +54,14 @@
|
||||
<name>log</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.log.LogConsumerConfiguration.class</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>byteArrayTextToString|logConsumer</functionDefinition>
|
||||
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>log-consumer</artifactId>
|
||||
<artifactId>spring-log-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>mongodb-consumer</artifactId>
|
||||
<artifactId>spring-mongodb-consumer</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.projectreactor</groupId>
|
||||
@@ -48,14 +48,13 @@
|
||||
<name>mongodb</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.mongo.MongoDbConsumerConfiguration.class
|
||||
</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>byteArrayTextToString|mongodbConsumer</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>mongodb-consumer</artifactId>
|
||||
<artifactId>spring-mongodb-consumer</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
|
||||
@@ -30,12 +30,12 @@ $$username$$:: $$the username to use when connecting to the broker.$$ *($$String
|
||||
|
||||
=== mqtt.consumer
|
||||
|
||||
$$async$$:: $$whether or not to use async sends.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$charset$$:: $$the charset used to convert a String payload to byte[].$$ *($$String$$, default: `$$UTF-8$$`)*
|
||||
$$client-id$$:: $$identifies the client.$$ *($$String$$, default: `$$stream.client.id.sink$$`)*
|
||||
$$qos$$:: $$the quality of service to use.$$ *($$Integer$$, default: `$$1$$`)*
|
||||
$$retained$$:: $$whether to set the 'retained' flag.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$topic$$:: $$the topic to which the sink will publish.$$ *($$String$$, default: `$$stream.mqtt$$`)*
|
||||
$$async$$:: $$Whether to use async sends.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$charset$$:: $$The charset used to convert a String payload to byte[].$$ *($$String$$, default: `$$UTF-8$$`)*
|
||||
$$client-id$$:: $$Identifies the client.$$ *($$String$$, default: `$$stream.client.id.sink$$`)*
|
||||
$$qos$$:: $$The quality of service to use.$$ *($$Integer$$, default: `$$1$$`)*
|
||||
$$retained$$:: $$Whether to set the 'retained' flag.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$topic$$:: $$The topic to which the sink will publish.$$ *($$String$$, default: `$$stream.mqtt$$`)*
|
||||
//end::configuration-properties[]
|
||||
|
||||
//end::ref-doc[]
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>mqtt-consumer</artifactId>
|
||||
<artifactId>spring-mqtt-consumer</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
@@ -49,13 +49,13 @@
|
||||
<name>mqtt</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.mqtt.MqttConsumerConfiguration.class</configClass>
|
||||
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>mqttConsumer</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>mqtt-consumer</artifactId>
|
||||
<artifactId>spring-mqtt-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -28,11 +28,6 @@
|
||||
<artifactId>postgresql</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-stream-test-binder</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
||||
@@ -13,14 +13,13 @@ The **$$rabbit$$** $$sink$$ has the following options:
|
||||
Properties grouped by prefix:
|
||||
|
||||
|
||||
=== rabbit
|
||||
=== rabbit.consumer
|
||||
|
||||
$$converter-bean-name$$:: $$The bean name for a custom message converter; if omitted, a SimpleMessageConverter is used. If 'jsonConverter', a Jackson2JsonMessageConverter bean will be created for you.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$exchange$$:: $$Exchange name - overridden by exchangeNameExpression, if supplied.$$ *($$String$$, default: `$$<empty string>$$`)*
|
||||
$$exchange-expression$$:: $$A SpEL expression that evaluates to an exchange name.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$headers-mapped-last$$:: $$When mapping headers for the outbound message, determine whether the headers are mapped before the message is converted, or afterwards.$$ *($$Boolean$$, default: `$$true$$`)*
|
||||
$$headers-mapped-last$$:: $$When mapping headers for the outbound message, determine whether the headers are mapped before the message is converted, or afterward.$$ *($$Boolean$$, default: `$$true$$`)*
|
||||
$$mapped-request-headers$$:: $$Headers that will be mapped.$$ *($$String[]$$, default: `$$[*]$$`)*
|
||||
$$own-connection$$:: $$When true, use a separate connection based on the boot properties.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$persistent-delivery-mode$$:: $$Default delivery mode when 'amqp_deliveryMode' header is not present, true for PERSISTENT.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$routing-key$$:: $$Routing key - overridden by routingKeyExpression, if supplied.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$routing-key-expression$$:: $$A SpEL expression that evaluates to a routing key.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
@@ -32,6 +31,7 @@ $$addresses$$:: $$Comma-separated list of addresses to which the client should c
|
||||
$$channel-rpc-timeout$$:: $$Continuation timeout for RPC calls in channels. Set it to zero to wait forever.$$ *($$Duration$$, default: `$$10m$$`)*
|
||||
$$connection-timeout$$:: $$Connection timeout. Set it to zero to wait forever.$$ *($$Duration$$, default: `$$<none>$$`)*
|
||||
$$host$$:: $$RabbitMQ host. Ignored if an address is set.$$ *($$String$$, default: `$$localhost$$`)*
|
||||
$$max-inbound-message-body-size$$:: $$Maximum size of the body of inbound (received) messages.$$ *($$DataSize$$, default: `$$64MB$$`)*
|
||||
$$password$$:: $$Login to authenticate against the broker.$$ *($$String$$, default: `$$guest$$`)*
|
||||
$$port$$:: $$RabbitMQ port. Ignored if an address is set. Default to 5672, or 5671 if SSL is enabled.$$ *($$Integer$$, default: `$$<none>$$`)*
|
||||
$$publisher-confirm-type$$:: $$Type of publisher confirms to use.$$ *($$ConfirmType$$, default: `$$<none>$$`, possible values: `SIMPLE`,`CORRELATED`,`NONE`)*
|
||||
|
||||
@@ -27,14 +27,9 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>rabbit-consumer</artifactId>
|
||||
<artifactId>spring-rabbit-consumer</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-stream-test-binder</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers</artifactId>
|
||||
<version>${testcontainers.version}</version>
|
||||
@@ -76,13 +71,13 @@
|
||||
<name>rabbit</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.rabbit.RabbitConsumerConfiguration.class</configClass>
|
||||
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>rabbitConsumer</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>rabbit-consumer</artifactId>
|
||||
<artifactId>spring-rabbit-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -14,11 +14,11 @@ Properties grouped by prefix:
|
||||
=== redis.consumer
|
||||
|
||||
$$key$$:: $$A literal key name to use when storing to a key.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$key-expression$$:: $$A SpEL expression to use for storing to a key.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$key-expression$$:: $$A SpEL expression to use for storing to a key.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$queue$$:: $$A literal queue name to use when storing in a queue.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$queue-expression$$:: $$A SpEL expression to use for queue.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$queue-expression$$:: $$A SpEL expression to use for queue.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$topic$$:: $$A literal topic name to use when publishing to a topic.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$topic-expression$$:: $$A SpEL expression to use for topic.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$topic-expression$$:: $$A SpEL expression to use for topic.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
|
||||
=== spring.data.redis
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>redis-consumer</artifactId>
|
||||
<artifactId>spring-redis-consumer</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
@@ -79,13 +79,13 @@
|
||||
<name>redis</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.redis.RedisConsumerConfiguration.class</configClass>
|
||||
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>redisConsumer</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>redis-consumer</artifactId>
|
||||
<artifactId>spring-redis-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2020-2022 the original author or authors.
|
||||
* Copyright 2020-2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -24,12 +24,10 @@ import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.WebApplicationType;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
import org.springframework.cloud.fn.consumer.redis.RedisConsumerConfiguration;
|
||||
import org.springframework.cloud.fn.consumer.redis.RedisTestContainerSupport;
|
||||
import org.springframework.cloud.stream.binder.test.InputDestination;
|
||||
import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.data.redis.support.collections.DefaultRedisList;
|
||||
import org.springframework.data.redis.support.collections.RedisList;
|
||||
@@ -47,32 +45,30 @@ public class RedisSinkTests implements RedisTestContainerSupport {
|
||||
|
||||
@Test
|
||||
public void testRedisSink() {
|
||||
String key = "foo";
|
||||
try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
|
||||
TestChannelBinderConfiguration
|
||||
.getCompleteConfiguration(RedisSinkTestApplication.class))
|
||||
.web(WebApplicationType.NONE)
|
||||
.run("--spring.cloud.function.definition=redisConsumer",
|
||||
"--spring.cloud.stream.bindings.redisConsumer-in-0.consumer.use-native-decoding=true",
|
||||
"--spring.data.redis.url=" + RedisTestContainerSupport.getUri(),
|
||||
"--redis.consumer.key=foo")) {
|
||||
"--redis.consumer.key=" + key)) {
|
||||
|
||||
//Setup
|
||||
String key = "foo";
|
||||
|
||||
final StringRedisTemplate redisTemplate = context.getBean(StringRedisTemplate.class);
|
||||
StringRedisTemplate redisTemplate = context.getBean(StringRedisTemplate.class);
|
||||
redisTemplate.delete(key);
|
||||
|
||||
RedisList<String> redisList = new DefaultRedisList<>(key, redisTemplate);
|
||||
List<String> list = new ArrayList<>();
|
||||
list.add("Manny");
|
||||
list.add("Moe");
|
||||
list.add("Jack");
|
||||
|
||||
//Execute
|
||||
Message<List<String>> message = new GenericMessage<>(list);
|
||||
|
||||
InputDestination source = context.getBean(InputDestination.class);
|
||||
source.send(message);
|
||||
|
||||
RedisList<String> redisList = new DefaultRedisList<>(key, redisTemplate);
|
||||
assertThat(redisList.size()).isEqualTo(3);
|
||||
assertThat(redisList.get(0)).isEqualTo("Manny");
|
||||
assertThat(redisList.get(1)).isEqualTo("Moe");
|
||||
@@ -81,7 +77,6 @@ public class RedisSinkTests implements RedisTestContainerSupport {
|
||||
}
|
||||
|
||||
@SpringBootApplication
|
||||
@Import(RedisConsumerConfiguration.class)
|
||||
public static class RedisSinkTestApplication {
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,7 +54,7 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>payload-converter-function</artifactId>
|
||||
<artifactId>spring-payload-converter-function</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2016-2023 the original author or authors.
|
||||
* Copyright 2016-2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -36,6 +36,7 @@ import org.springframework.integration.router.AbstractMappingMessageRouter;
|
||||
import org.springframework.integration.router.AbstractMessageRouter;
|
||||
import org.springframework.integration.router.ExpressionEvaluatingRouter;
|
||||
import org.springframework.integration.router.MethodInvokingRouter;
|
||||
import org.springframework.integration.scripting.dsl.ScriptSpec;
|
||||
import org.springframework.integration.scripting.dsl.Scripts;
|
||||
import org.springframework.lang.Nullable;
|
||||
import org.springframework.messaging.Message;
|
||||
@@ -93,12 +94,11 @@ public class RouterSinkConfiguration {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty("router.script")
|
||||
public MessageProcessor<?> scriptProcessor() {
|
||||
public ScriptSpec scriptProcessor() {
|
||||
return Scripts.processor(this.properties.getScript())
|
||||
.lang("groovy")
|
||||
.refreshCheckDelay(this.properties.getRefreshDelay())
|
||||
.variables(obtainScriptVariables(this.properties))
|
||||
.get();
|
||||
.variables(obtainScriptVariables(this.properties));
|
||||
}
|
||||
|
||||
private static Map<String, Object> obtainScriptVariables(RouterSinkProperties properties) {
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>rsocket-consumer</artifactId>
|
||||
<artifactId>spring-rsocket-consumer</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.projectreactor</groupId>
|
||||
@@ -53,12 +53,13 @@
|
||||
<name>rsocket</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.rsocket.RsocketConsumerConfiguration.class</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>rsocketConsumer</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>rsocket-consumer</artifactId>
|
||||
<artifactId>spring-rsocket-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2020-2020 the original author or authors.
|
||||
* Copyright 2020-2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -16,13 +16,11 @@
|
||||
|
||||
package org.springframework.cloud.stream.app.rsocket.sink;
|
||||
|
||||
import java.util.function.Function;
|
||||
import java.time.Duration;
|
||||
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import reactor.core.publisher.Flux;
|
||||
import reactor.core.publisher.Mono;
|
||||
import reactor.core.publisher.ReplayProcessor;
|
||||
import reactor.core.publisher.Sinks;
|
||||
import reactor.test.StepVerifier;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -36,11 +34,9 @@ import org.springframework.cloud.fn.consumer.rsocket.RsocketConsumerConfiguratio
|
||||
import org.springframework.cloud.stream.binder.test.InputDestination;
|
||||
import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.integration.support.MessageBuilder;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.messaging.handler.annotation.MessageMapping;
|
||||
import org.springframework.messaging.support.GenericMessage;
|
||||
import org.springframework.stereotype.Controller;
|
||||
import org.springframework.test.annotation.DirtiesContext;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
@@ -48,7 +44,7 @@ import org.springframework.test.util.ReflectionTestUtils;
|
||||
/**
|
||||
* @author Soby Chacko
|
||||
*/
|
||||
@SpringBootTest(properties = {"spring.rsocket.server.port=0"}, classes = RSocketSinkTests.RSocketserverApplication.class)
|
||||
@SpringBootTest(properties = {"spring.rsocket.server.port=0"}, classes = RSocketSinkTests.RSocketServerApplication.class)
|
||||
@DirtiesContext
|
||||
public class RSocketSinkTests {
|
||||
|
||||
@@ -71,15 +67,11 @@ public class RSocketSinkTests {
|
||||
final int port = server.address().getPort();
|
||||
|
||||
applicationContextRunner.withPropertyValues(
|
||||
"spring.cloud.function.definition=rsocketConsumer",
|
||||
"rsocket.consumer.port=" + port,
|
||||
"rsocket.consumer.route=test-route")
|
||||
"spring.cloud.function.definition=rsocketFunctionConsumer",
|
||||
"rsocket.consumer.port=" + port,
|
||||
"rsocket.consumer.route=test-route")
|
||||
.run(context -> {
|
||||
Function<Flux<Message<?>>, Mono<Void>> rsocketConsumer = context.getBean("rsocketConsumer", Function.class);
|
||||
rsocketConsumer.apply(Flux.just(new GenericMessage<>("Hello RSocket")))
|
||||
.subscribe();
|
||||
|
||||
final StepVerifier stepVerifier = StepVerifier.create(RSocketserverApplication.fireForgetPayloads)
|
||||
final StepVerifier stepVerifier = StepVerifier.create(RSocketServerApplication.fireForgetPayloads.asMono())
|
||||
.expectNext("Hello RSocket")
|
||||
.thenCancel()
|
||||
.verifyLater();
|
||||
@@ -88,25 +80,24 @@ public class RSocketSinkTests {
|
||||
InputDestination source = context.getBean(InputDestination.class);
|
||||
source.send(message);
|
||||
|
||||
stepVerifier.verify();
|
||||
stepVerifier.verify(Duration.ofSeconds(10));
|
||||
});
|
||||
}
|
||||
|
||||
@EnableAutoConfiguration
|
||||
@EnableAutoConfiguration(exclude = RsocketConsumerConfiguration.class)
|
||||
@SpringBootConfiguration
|
||||
@Controller
|
||||
static class RSocketserverApplication {
|
||||
static final ReplayProcessor<String> fireForgetPayloads = ReplayProcessor.create();
|
||||
static class RSocketServerApplication {
|
||||
static final Sinks.One<String> fireForgetPayloads = Sinks.one();
|
||||
|
||||
@MessageMapping("test-route")
|
||||
void someMethod(String payload) {
|
||||
this.fireForgetPayloads.onNext(payload);
|
||||
this.fireForgetPayloads.tryEmitValue(payload);
|
||||
}
|
||||
}
|
||||
|
||||
@EnableAutoConfiguration
|
||||
@SpringBootConfiguration
|
||||
@Import(RsocketConsumerConfiguration.class)
|
||||
static class RsocketSinkTestApplication {
|
||||
|
||||
}
|
||||
|
||||
@@ -44,6 +44,7 @@ $$static$$:: $$<documentation missing>$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$accelerate-mode-enabled$$:: $$Option to enable using the accelerate endpoint when accessing S3. Accelerate endpoints allow faster transfer of objects by using Amazon CloudFront's globally distributed edge locations.$$ *($$Boolean$$, default: `$$<none>$$`)*
|
||||
$$checksum-validation-enabled$$:: $$Option to disable doing a validation of the checksum of an object stored in S3.$$ *($$Boolean$$, default: `$$<none>$$`)*
|
||||
$$chunked-encoding-enabled$$:: $$Option to enable using chunked encoding when signing the request payload for {@link software.amazon.awssdk.services.s3.model.PutObjectRequest} and {@link software.amazon.awssdk.services.s3.model.UploadPartRequest}.$$ *($$Boolean$$, default: `$$<none>$$`)*
|
||||
$$cross-region-enabled$$:: $$Enables cross-region bucket access.$$ *($$Boolean$$, default: `$$<none>$$`)*
|
||||
$$endpoint$$:: $$Overrides the default endpoint.$$ *($$URI$$, default: `$$<none>$$`)*
|
||||
$$path-style-access-enabled$$:: $$Option to enable using path style access for accessing S3 objects instead of DNS style access. DNS style access is preferred as it will result in better load balancing when accessing S3.$$ *($$Boolean$$, default: `$$<none>$$`)*
|
||||
$$region$$:: $$Overrides the default region.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>s3-consumer</artifactId>
|
||||
<artifactId>spring-s3-consumer</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.integration</groupId>
|
||||
@@ -58,13 +58,13 @@
|
||||
<name>s3</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.s3.AwsS3ConsumerConfiguration.class</configClass>
|
||||
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>s3Consumer</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>s3-consumer</artifactId>
|
||||
<artifactId>spring-s3-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2016-2023 the original author or authors.
|
||||
* Copyright 2016-2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -17,6 +17,7 @@
|
||||
package org.springframework.cloud.stream.app.s3.sink;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.file.Path;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
@@ -40,7 +41,7 @@ import org.springframework.beans.DirectFieldAccessor;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.cloud.fn.consumer.s3.AwsS3ConsumerConfiguration;
|
||||
import org.springframework.boot.test.mock.mockito.MockBean;
|
||||
import org.springframework.cloud.stream.binder.test.InputDestination;
|
||||
import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
@@ -54,7 +55,6 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.BDDMockito.willReturn;
|
||||
import static org.mockito.Mockito.atLeastOnce;
|
||||
import static org.mockito.Mockito.spy;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE,
|
||||
@@ -62,6 +62,7 @@ import static org.mockito.Mockito.verify;
|
||||
"spring.cloud.aws.credentials.accessKey=" + AwsS3SinkTests.AWS_ACCESS_KEY,
|
||||
"spring.cloud.aws.credentials.secretKey=" + AwsS3SinkTests.AWS_SECRET_KEY,
|
||||
"spring.cloud.aws.region.static=" + AwsS3SinkTests.AWS_REGION,
|
||||
"spring.cloud.aws.s3.endpoint=s3://test.endpoint",
|
||||
"s3.consumer.bucket=" + AwsS3SinkTests.S3_BUCKET,
|
||||
"s3.consumer.acl=PUBLIC_READ_WRITE"})
|
||||
@DirtiesContext
|
||||
@@ -78,7 +79,7 @@ public class AwsS3SinkTests {
|
||||
@TempDir
|
||||
protected static Path temporaryRemoteFolder;
|
||||
|
||||
@Autowired
|
||||
@MockBean
|
||||
private S3AsyncClient amazonS3;
|
||||
|
||||
@Autowired
|
||||
@@ -95,8 +96,6 @@ public class AwsS3SinkTests {
|
||||
|
||||
@BeforeEach
|
||||
public void setupTest() {
|
||||
S3AsyncClient amazonS3 = spy(this.amazonS3);
|
||||
|
||||
willReturn(CompletableFuture.completedFuture(PutObjectResponse.builder().build()))
|
||||
.given(amazonS3)
|
||||
.putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class));
|
||||
@@ -133,7 +132,7 @@ public class AwsS3SinkTests {
|
||||
|
||||
AsyncRequestBody asyncRequestBody = asyncRequestBodyArgumentCaptor.getValue();
|
||||
StepVerifier.create(asyncRequestBody)
|
||||
.assertNext(buffer -> assertThat(buffer.array()).isEmpty())
|
||||
.assertNext(buffer -> assertThat(buffer).isEqualTo(ByteBuffer.allocate(0)))
|
||||
.expectComplete()
|
||||
.verify();
|
||||
|
||||
@@ -141,7 +140,7 @@ public class AwsS3SinkTests {
|
||||
}
|
||||
|
||||
@SpringBootApplication
|
||||
@Import({AwsS3ConsumerConfiguration.class, TestChannelBinderConfiguration.class})
|
||||
@Import(TestChannelBinderConfiguration.class)
|
||||
public static class SampleConfiguration {
|
||||
|
||||
@Bean
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>sftp-consumer</artifactId>
|
||||
<artifactId>spring-sftp-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
@@ -43,13 +43,13 @@
|
||||
<name>sftp</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.sftp.SftpConsumerConfiguration.class</configClass>
|
||||
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>sftpConsumer</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>sftp-consumer</artifactId>
|
||||
<artifactId>spring-sftp-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -23,11 +23,11 @@ $$host$$:: $$The host to which this sink will connect.$$ *($$String$$, default:
|
||||
|
||||
=== tcp
|
||||
|
||||
$$nio$$:: $$Whether or not to use NIO.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$nio$$:: $$Whether to use NIO.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$port$$:: $$The port on which to listen; 0 for the OS to choose a port.$$ *($$Integer$$, default: `$$1234$$`)*
|
||||
$$reverse-lookup$$:: $$Perform a reverse DNS lookup on the remote IP Address; if false, just the IP address is included in the message headers.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$socket-timeout$$:: $$The timeout (ms) before closing the socket when no data is received.$$ *($$Integer$$, default: `$$120000$$`)*
|
||||
$$use-direct-buffers$$:: $$Whether or not to use direct buffers.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$use-direct-buffers$$:: $$Whether to use direct buffers.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
//end::configuration-properties[]
|
||||
|
||||
== Available Encoders
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>tcp-consumer</artifactId>
|
||||
<artifactId>spring-tcp-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
@@ -43,12 +43,13 @@
|
||||
<name>tcp</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.tcp.TcpConsumerConfiguration.class</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>tcpConsumer</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>tcp-consumer</artifactId>
|
||||
<artifactId>spring-tcp-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -8,7 +8,7 @@ Sink that will count messages and log the observed throughput at a selected inte
|
||||
The **$$throughput$$** $$sink$$ has the following options:
|
||||
|
||||
//tag::configuration-properties[]
|
||||
$$throughput.report-every-ms$$:: $$how often to report.$$ *($$Integer$$, default: `$$<none>$$`)*
|
||||
$$throughput.report-every-ms$$:: $$how often to report.$$ *($$Integer$$, default: `$$1000$$`)*
|
||||
//end::configuration-properties[]
|
||||
|
||||
//end::ref-doc[]
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>twitter-consumer</artifactId>
|
||||
<artifactId>spring-twitter-consumer</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.awaitility</groupId>
|
||||
@@ -33,13 +33,6 @@
|
||||
<dependency>
|
||||
<groupId>org.mock-server</groupId>
|
||||
<artifactId>mockserver-netty</artifactId>
|
||||
<version>${mockserver.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mock-server</groupId>
|
||||
<artifactId>mockserver-client-java</artifactId>
|
||||
<version>${mockserver.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
@@ -66,14 +59,14 @@
|
||||
<name>twitter-message</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.twitter.message.TwitterMessageConsumerConfiguration.class</configClass>
|
||||
<functionDefinition>byteArrayTextToString|sendDirectMessageConsumer</functionDefinition>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>byteArrayTextToString|twitterSendMessageConsumer</functionDefinition>
|
||||
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>twitter-consumer</artifactId>
|
||||
<artifactId>spring-twitter-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2020-2020 the original author or authors.
|
||||
* Copyright 2020-2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -36,15 +36,12 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
import org.springframework.cloud.fn.common.twitter.TwitterConnectionProperties;
|
||||
import org.springframework.cloud.fn.common.twitter.util.TwitterTestUtils;
|
||||
import org.springframework.cloud.fn.consumer.twitter.message.TwitterMessageConsumerConfiguration;
|
||||
import org.springframework.cloud.stream.binder.test.InputDestination;
|
||||
import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.messaging.support.GenericMessage;
|
||||
import org.springframework.test.util.TestSocketUtils;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockserver.matchers.Times.unlimited;
|
||||
@@ -54,21 +51,18 @@ import static org.mockserver.verify.VerificationTimes.once;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
* @author Artem Bilan
|
||||
*/
|
||||
public class TwitterMessageSinkIntegrationTests {
|
||||
|
||||
private static final String MOCK_SERVER_IP = "127.0.0.1";
|
||||
|
||||
private static final Integer MOCK_SERVER_PORT = TestSocketUtils.findAvailableTcpPort();
|
||||
|
||||
private static ClientAndServer mockServer;
|
||||
|
||||
private static MockServerClient mockClient;
|
||||
|
||||
@BeforeEach
|
||||
public void startMockServer() {
|
||||
mockServer = ClientAndServer.startClientAndServer(MOCK_SERVER_PORT);
|
||||
mockClient = new MockServerClient(MOCK_SERVER_IP, MOCK_SERVER_PORT);
|
||||
mockServer = ClientAndServer.startClientAndServer();
|
||||
mockClient = new MockServerClient("localhost", mockServer.getPort());
|
||||
|
||||
mockClient
|
||||
.when(
|
||||
@@ -111,7 +105,7 @@ public class TwitterMessageSinkIntegrationTests {
|
||||
try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
|
||||
TestChannelBinderConfiguration.getCompleteConfiguration(TestTwitterMessageSinkApplication.class))
|
||||
.web(WebApplicationType.NONE)
|
||||
.run("--spring.cloud.function.definition=byteArrayTextToString|sendDirectMessageConsumer",
|
||||
.run("--spring.cloud.function.definition=byteArrayTextToString|twitterSendMessageConsumer",
|
||||
|
||||
"--twitter.message.update.screenName='user666'",
|
||||
|
||||
@@ -149,7 +143,7 @@ public class TwitterMessageSinkIntegrationTests {
|
||||
try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
|
||||
TestChannelBinderConfiguration.getCompleteConfiguration(TestTwitterMessageSinkApplication.class))
|
||||
.web(WebApplicationType.NONE)
|
||||
.run("--spring.cloud.function.definition=byteArrayTextToString|sendDirectMessageConsumer",
|
||||
.run("--spring.cloud.function.definition=byteArrayTextToString|twitterSendMessageConsumer",
|
||||
|
||||
"--twitter.message.update.userId='1075751718749659136'",
|
||||
|
||||
@@ -178,7 +172,7 @@ public class TwitterMessageSinkIntegrationTests {
|
||||
try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
|
||||
TestChannelBinderConfiguration.getCompleteConfiguration(TestTwitterMessageSinkApplication.class))
|
||||
.web(WebApplicationType.NONE)
|
||||
.run("--spring.cloud.function.definition=byteArrayTextToString|sendDirectMessageConsumer",
|
||||
.run("--spring.cloud.function.definition=byteArrayTextToString|twitterSendMessageConsumer",
|
||||
|
||||
"--twitter.message.update.userId=headers['user']",
|
||||
"--twitter.message.update.text=payload.concat(\" with suffix \")",
|
||||
@@ -192,8 +186,8 @@ public class TwitterMessageSinkIntegrationTests {
|
||||
InputDestination source = context.getBean(InputDestination.class);
|
||||
assertThat(source).isNotNull();
|
||||
|
||||
Map<String, String> headers = Collections.singletonMap("user", "1075751718749659136");
|
||||
source.send(new GenericMessage("hello".getBytes(StandardCharsets.UTF_8), headers));
|
||||
Map<String, Object> headers = Collections.singletonMap("user", "1075751718749659136");
|
||||
source.send(new GenericMessage<>("hello".getBytes(StandardCharsets.UTF_8), headers));
|
||||
|
||||
mockClient.verify(request()
|
||||
.withMethod("POST")
|
||||
@@ -208,7 +202,6 @@ public class TwitterMessageSinkIntegrationTests {
|
||||
|
||||
@SpringBootConfiguration
|
||||
@EnableAutoConfiguration
|
||||
@Import(TwitterMessageConsumerConfiguration.class)
|
||||
public static class TestTwitterMessageSinkApplication {
|
||||
|
||||
@Bean
|
||||
@@ -218,10 +211,11 @@ public class TwitterMessageSinkIntegrationTests {
|
||||
|
||||
Function<TwitterConnectionProperties, ConfigurationBuilder> mockedConfiguration =
|
||||
toConfigurationBuilder.andThen(
|
||||
new TwitterTestUtils().mockTwitterUrls(
|
||||
String.format("http://%s:%s", MOCK_SERVER_IP, MOCK_SERVER_PORT)));
|
||||
new TwitterTestUtils().mockTwitterUrls("http://localhost:" + mockServer.getPort()));
|
||||
|
||||
return mockedConfiguration.apply(properties).build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -21,9 +21,9 @@ Properties grouped by prefix:
|
||||
|
||||
=== twitter.update
|
||||
|
||||
$$attachment-url$$:: $$(SpEL expression) In order for a URL to not be counted in the text body of an extended Tweet, provide a URL as a Tweet attachment. This URL must be a Tweet permalink, or Direct Message deep link. Arbitrary, non-Twitter URLs must remain in the text text. URLs passed to the attachment_url parameter not matching either a Tweet permalink or Direct Message deep link will fail at Tweet creation and cause an exception.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$display-coordinates$$:: $$(SpEL expression) Whether or not to put a pin on the exact coordinates a Tweet has been sent from.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$in-reply-to-status-id$$:: $$(SpEL expression) The ID of an existing text that the update is in reply to. Note: This parameter will be ignored unless the author of the Tweet this parameter references is mentioned within the text text. Therefore, you must include @username, where username is the author of the referenced Tweet, within the update. When inReplyToStatusId is set the auto_populate_reply_metadata is automatically set as well. Later ensures that leading @mentions will be looked up from the original Tweet, and added to the new Tweet from there. This wil append @mentions into the metadata of an extended Tweet as a reply chain grows, until the limit on @mentions is reached. In cases where the original Tweet has been deleted, the reply will fail.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$attachment-url$$:: $$(SpEL expression) In order for a URL to not be counted in the text body of an extended Tweet, provide a URL as a Tweet attachment. This URL must be a Tweet permalink, or Direct Message deep link. Arbitrary, non-Twitter URLs must remain in the text. URLs passed to the attachment_url parameter not matching either a Tweet permalink or Direct Message deep link will fail at Tweet creation and cause an exception.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$display-coordinates$$:: $$(SpEL expression) Whether to put a pin on the exact coordinates a Tweet has been sent from.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$in-reply-to-status-id$$:: $$(SpEL expression) The ID of an existing text that the update is in reply to. Note: This parameter will be ignored unless the author of the Tweet this parameter references is mentioned within the text. Therefore, you must include @username, where username is the author of the referenced Tweet, within the update. When inReplyToStatusId is set the auto_populate_reply_metadata is automatically set as well. Later ensures that leading @mentions will be looked up from the original Tweet, and added to the new Tweet from there. This wil append @mentions into the metadata of an extended Tweet as a reply chain grows, until the limit on @mentions is reached. In cases where the original Tweet has been deleted, the reply will fail.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$media-ids$$:: $$(SpEL expression) A comma-delimited list of media_ids to associate with the Tweet. You may include up to 4 photos or 1 animated GIF or 1 video in a Tweet. See Uploading Media for further details on uploading media.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$place-id$$:: $$(SpEL expression) A place in the world.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$text$$:: $$(SpEL expression) The text of the text update. URL encode as necessary. t.co link wrapping will affect character counts. Defaults to message's payload$$ *($$Expression$$, default: `$$payload$$`)*
|
||||
@@ -31,7 +31,7 @@ $$text$$:: $$(SpEL expression) The text of the text update. URL encode as necess
|
||||
=== twitter.update.location
|
||||
|
||||
$$lat$$:: $$The latitude of the location this Tweet refers to. This parameter will be ignored unless it is inside the range -90.0 to +90.0 (North is positive) inclusive. It will also be ignored if there is no corresponding long parameter.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$lon$$:: $$The longitude of the location this Tweet refers to. The valid ranges for longitude are -180.0 to +180.0 (East is positive) inclusive. This parameter will be ignored if outside that range, if it is not a number, if geo_enabled is disabled, or if there no corresponding lat parameter.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
$$lon$$:: $$The longitude of the location this Tweet refers to. The valid ranges for longitude are -180.0 to +180.0 (East is positive) inclusive. This parameter will be ignored if outside that range, if it is not a number, if geo_enabled is disabled, or if there is no corresponding lat parameter.$$ *($$Expression$$, default: `$$<none>$$`)*
|
||||
//end::configuration-properties[]
|
||||
|
||||
//end::ref-doc[]
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>twitter-consumer</artifactId>
|
||||
<artifactId>spring-twitter-consumer</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.awaitility</groupId>
|
||||
@@ -33,13 +33,6 @@
|
||||
<dependency>
|
||||
<groupId>org.mock-server</groupId>
|
||||
<artifactId>mockserver-netty</artifactId>
|
||||
<version>${mockserver.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mock-server</groupId>
|
||||
<artifactId>mockserver-client-java</artifactId>
|
||||
<version>${mockserver.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
@@ -66,14 +59,14 @@
|
||||
<name>twitter-update</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.twitter.status.update.TwitterUpdateConsumerConfiguration.class</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>byteArrayTextToString|twitterStatusUpdateConsumer</functionDefinition>
|
||||
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>twitter-consumer</artifactId>
|
||||
<artifactId>spring-twitter-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
package org.springframework.cloud.stream.app.sink.twitter.update;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Function;
|
||||
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
@@ -34,16 +33,13 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
import org.springframework.cloud.fn.common.twitter.TwitterConnectionProperties;
|
||||
import org.springframework.cloud.fn.common.twitter.util.TwitterTestUtils;
|
||||
import org.springframework.cloud.fn.consumer.twitter.status.update.TwitterUpdateConsumerConfiguration;
|
||||
import org.springframework.cloud.fn.consumer.twitter.status.update.TwitterUpdateConsumerProperties;
|
||||
import org.springframework.cloud.stream.binder.test.InputDestination;
|
||||
import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration;
|
||||
import org.springframework.context.ConfigurableApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.messaging.support.GenericMessage;
|
||||
import org.springframework.test.util.TestSocketUtils;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockserver.matchers.Times.unlimited;
|
||||
@@ -53,21 +49,19 @@ import static org.mockserver.verify.VerificationTimes.once;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
* @author Artem Bilan
|
||||
*/
|
||||
public class TwitterUpdateSinkIntegrationTests {
|
||||
|
||||
private static final String MOCK_SERVER_IP = "127.0.0.1";
|
||||
|
||||
private static final Integer MOCK_SERVER_PORT = TestSocketUtils.findAvailableTcpPort();
|
||||
|
||||
private static ClientAndServer mockServer;
|
||||
|
||||
private static MockServerClient mockClient;
|
||||
|
||||
@BeforeAll
|
||||
public static void startMockServer() {
|
||||
mockServer = ClientAndServer.startClientAndServer(MOCK_SERVER_PORT);
|
||||
mockClient = new MockServerClient(MOCK_SERVER_IP, MOCK_SERVER_PORT);
|
||||
mockServer = ClientAndServer.startClientAndServer();
|
||||
mockClient = new MockServerClient("localhost", mockServer.getPort());
|
||||
|
||||
mockClient
|
||||
.when(
|
||||
request().withMethod("POST").withPath("/statuses/update.json"),
|
||||
@@ -75,8 +69,7 @@ public class TwitterUpdateSinkIntegrationTests {
|
||||
.respond(response()
|
||||
.withStatusCode(200)
|
||||
.withHeader("Content-Type", "application/json; charset=utf-8")
|
||||
.withBody(TwitterTestUtils.asString("classpath:/response/update_test_1.json"))
|
||||
.withDelay(TimeUnit.SECONDS, 1));
|
||||
.withBody(TwitterTestUtils.asString("classpath:/response/update_test_1.json")));
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
@@ -211,7 +204,6 @@ public class TwitterUpdateSinkIntegrationTests {
|
||||
|
||||
@SpringBootConfiguration
|
||||
@EnableAutoConfiguration
|
||||
@Import(TwitterUpdateConsumerConfiguration.class)
|
||||
public static class TestTwitterUpdateSinkApplication {
|
||||
|
||||
@Bean
|
||||
@@ -221,10 +213,11 @@ public class TwitterUpdateSinkIntegrationTests {
|
||||
|
||||
Function<TwitterConnectionProperties, ConfigurationBuilder> mockedConfiguration =
|
||||
toConfigurationBuilder.andThen(
|
||||
new TwitterTestUtils().mockTwitterUrls(
|
||||
String.format("http://%s:%s", MOCK_SERVER_IP, MOCK_SERVER_PORT)));
|
||||
new TwitterTestUtils().mockTwitterUrls("http://localhost:" + mockServer.getPort()));
|
||||
|
||||
return mockedConfiguration.apply(properties).build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -17,11 +17,11 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>wavefront-consumer</artifactId>
|
||||
<artifactId>spring-wavefront-consumer</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>function-test-support</artifactId>
|
||||
<artifactId>spring-function-test-support</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
@@ -48,16 +48,14 @@
|
||||
<name>wavefront</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>
|
||||
org.springframework.cloud.fn.consumer.wavefront.WavefrontConsumerConfiguration.class
|
||||
</configClass>
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>wavefrontConsumer</functionDefinition>
|
||||
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>wavefront-consumer</artifactId>
|
||||
<artifactId>spring-wavefront-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -7,11 +7,11 @@ A simple Websocket Sink implementation.
|
||||
The following options are supported:
|
||||
|
||||
//tag::configuration-properties[]
|
||||
$$websocket.consumer.log-level$$:: $$the logLevel for netty channels. Default is <tt>WARN</tt>$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$websocket.consumer.path$$:: $$the path on which a WebsocketSink consumer needs to connect. Default is <tt>/websocket</tt>$$ *($$String$$, default: `$$/websocket$$`)*
|
||||
$$websocket.consumer.port$$:: $$the port on which the Netty server listens. Default is <tt>9292</tt>$$ *($$Integer$$, default: `$$9292$$`)*
|
||||
$$websocket.consumer.ssl$$:: $$whether or not to create a {@link io.netty.handler.ssl.SslContext}.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$websocket.consumer.threads$$:: $$the number of threads for the Netty {@link io.netty.channel.EventLoopGroup}. Default is <tt>1</tt>$$ *($$Integer$$, default: `$$1$$`)*
|
||||
$$websocket.consumer.log-level$$:: $$The logLevel for netty channels. Default is <tt>WARN</tt>$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$websocket.consumer.path$$:: $$The path on which a WebsocketSink consumer needs to connect. Default is <tt>/websocket</tt>$$ *($$String$$, default: `$$/websocket$$`)*
|
||||
$$websocket.consumer.port$$:: $$The port on which the Netty server listens. Default is <tt>9292</tt>$$ *($$Integer$$, default: `$$9292$$`)*
|
||||
$$websocket.consumer.ssl$$:: $$Whether to create a {@link io.netty.handler.ssl.SslContext}.$$ *($$Boolean$$, default: `$$false$$`)*
|
||||
$$websocket.consumer.threads$$:: $$The number of threads for the Netty {@link io.netty.channel.EventLoopGroup}. Default is <tt>1</tt>$$ *($$Integer$$, default: `$$1$$`)*
|
||||
//end::configuration-properties[]
|
||||
|
||||
== Examples
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>websocket-consumer</artifactId>
|
||||
<artifactId>spring-websocket-consumer</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
@@ -26,7 +26,7 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>function-test-support</artifactId>
|
||||
<artifactId>spring-function-test-support</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
@@ -53,13 +53,13 @@
|
||||
<name>websocket</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.websocket.WebsocketConsumerConfiguration.class</configClass>
|
||||
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>websocketConsumer</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>websocket-consumer</artifactId>
|
||||
<artifactId>spring-websocket-consumer</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</maven>
|
||||
|
||||
@@ -29,8 +29,8 @@ $$chat-to$$:: $$XMPP handle to send message to.$$ *($$String$$, default: `$$<non
|
||||
|
||||
$$host$$:: $$XMPP Host server to connect to.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$password$$:: $$The Password for the connected user.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$port$$:: $$Port for connecting to the host. - Default Client Port: 5222$$ *($$Integer$$, default: `$$5222$$`)*
|
||||
$$resource$$:: $$The Resource to bind to on the XMPP Host. - Can be empty, server will generate one if not set$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$port$$:: $$Port for connecting to the host. - Default Client Port: 5222$$ *($$Integer$$, default: `$$5222$$`)*
|
||||
$$resource$$:: $$The Resource to bind to on the XMPP Host. - Can be empty, server will generate one if not set$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$security-mode$$:: $$<documentation missing>$$ *($$SecurityMode$$, default: `$$<none>$$`, possible values: `required`,`ifpossible`,`disabled`)*
|
||||
$$service-name$$:: $$The Service Name to set for the XMPP Domain.$$ *($$String$$, default: `$$<none>$$`)*
|
||||
$$subscription-mode$$:: $$<documentation missing>$$ *($$SubscriptionMode$$, default: `$$<none>$$`, possible values: `accept_all`,`reject_all`,`manual`)*
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
<!-- Spring Cloud Function dependencies -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>xmpp-consumer</artifactId>
|
||||
<artifactId>spring-xmpp-consumer</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- Test dependencies -->
|
||||
@@ -37,7 +37,7 @@
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>function-test-support</artifactId>
|
||||
<artifactId>spring-function-test-support</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
@@ -66,14 +66,13 @@
|
||||
<name>xmpp</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.xmpp.XmppConsumerConfiguration.class
|
||||
</configClass>
|
||||
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>xmppConsumer</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>xmpp-consumer</artifactId>
|
||||
<artifactId>spring-xmpp-consumer</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.stream.app</groupId>
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
<!-- Spring Cloud Function dependencies -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>zeromq-consumer</artifactId>
|
||||
<artifactId>spring-zeromq-consumer</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- Test dependencies -->
|
||||
@@ -58,14 +58,13 @@
|
||||
<name>zeromq</name>
|
||||
<type>sink</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.consumer.zeromq.ZeroMqConsumerConfiguration.class
|
||||
</configClass>
|
||||
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>zeromqConsumer</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>zeromq-consumer</artifactId>
|
||||
<artifactId>spring-zeromq-consumer</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.stream.app</groupId>
|
||||
|
||||
@@ -24,14 +24,14 @@ Properties grouped by prefix:
|
||||
=== debezium
|
||||
|
||||
$$debezium-native-configuration$$:: $$<documentation missing>$$ *($$Properties$$, default: `$$<none>$$`)*
|
||||
$$header-format$$:: $${@link ChangeEvent} header format. Defaults to 'JSON'.$$ *($$DebeziumFormat$$, default: `$$<none>$$`, possible values: `JSON`,`AVRO`,`PROTOBUF`)*
|
||||
$$offset-commit-policy$$:: $$The policy that defines when the offsets should be committed to offset storage.$$ *($$DebeziumOffsetCommitPolicy$$, default: `$$<none>$$`, possible values: `ALWAYS`,`PERIODIC`,`DEFAULT`)*
|
||||
$$payload-format$$:: $${@link ChangeEvent} Key and Payload formats. Defaults to 'JSON'.$$ *($$DebeziumFormat$$, default: `$$<none>$$`, possible values: `JSON`,`AVRO`,`PROTOBUF`)*
|
||||
$$payload-format$$:: $${@code io.debezium.engine.ChangeEvent} Key and Payload formats. Defaults to 'JSON'.$$ *($$DebeziumFormat$$, default: `$$<none>$$`, possible values: `JSON`,`AVRO`,`PROTOBUF`)*
|
||||
$$properties$$:: $$Spring pass-trough wrapper for debezium configuration properties. All properties with a 'debezium.properties.*' prefix are native Debezium properties.$$ *($$Map<String, String>$$, default: `$$<none>$$`)*
|
||||
|
||||
=== debezium.supplier
|
||||
|
||||
$$copy-headers$$:: $$Copy Change Event headers into Message headers.$$ *($$Boolean$$, default: `$$true$$`)*
|
||||
$$enable-empty-payload$$:: $$Enable support for tombstone (aka delete) messages.$$ *($$Boolean$$, default: `$$true$$`)*
|
||||
$$header-names-to-map$$:: $$Patterns for {@code ChangeEvent.headers()} to map.$$ *($$String[]$$, default: `$$[*]$$`)*
|
||||
//end::configuration-properties[]
|
||||
|
||||
==== Event flattening configuration
|
||||
|
||||
@@ -22,15 +22,13 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>debezium-supplier</artifactId>
|
||||
<version>${java-functions.version}</version>
|
||||
<artifactId>spring-debezium-supplier</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- TEST -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>function-test-support</artifactId>
|
||||
<version>${java-functions.version}</version>
|
||||
<artifactId>spring-function-test-support</artifactId>
|
||||
<scope>test</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
@@ -159,7 +157,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>debezium-supplier</artifactId>
|
||||
<artifactId>spring-debezium-supplier</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.stream.app</groupId>
|
||||
|
||||
@@ -18,6 +18,7 @@ package org.springframework.cloud.stream.app.source.debezium.integration;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
@@ -26,19 +27,15 @@ import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
|
||||
import org.springframework.boot.test.context.FilteredClassLoader;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
import org.springframework.boot.test.context.runner.ContextConsumer;
|
||||
import org.springframework.cloud.fn.common.debezium.DebeziumProperties;
|
||||
import org.springframework.cloud.fn.supplier.debezium.DebeziumReactiveConsumerConfiguration;
|
||||
import org.springframework.cloud.stream.binder.test.OutputDestination;
|
||||
import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.kafka.support.KafkaNull;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.test.jdbc.JdbcTestUtils;
|
||||
import org.springframework.util.ClassUtils;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@@ -106,10 +103,6 @@ public class DebeziumDeleteHandlingIntegrationTest {
|
||||
"debezium.properties.transforms.unwrap.delete.handling.mode=rewrite,debezium.properties.transforms.unwrap.drop.tombstones=false"
|
||||
})
|
||||
public void handleRecordDeletions(String properties) {
|
||||
contextRunner.withPropertyValues(properties.split(","))
|
||||
.withClassLoader(new FilteredClassLoader(KafkaNull.class)) // Remove Kafka from the
|
||||
.run(consumer);
|
||||
|
||||
contextRunner.withPropertyValues(properties.split(","))
|
||||
.run(consumer);
|
||||
}
|
||||
@@ -123,10 +116,6 @@ public class DebeziumDeleteHandlingIntegrationTest {
|
||||
JdbcTemplate jdbcTemplate = context.getBean(JdbcTemplate.class);
|
||||
DebeziumProperties props = context.getBean(DebeziumProperties.class);
|
||||
|
||||
boolean isKafkaPresent = ClassUtils.isPresent(
|
||||
DebeziumReactiveConsumerConfiguration.ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL,
|
||||
context.getClassLoader());
|
||||
|
||||
String deleteHandlingMode = props.getProperties().get("transforms.unwrap.delete.handling.mode");
|
||||
String isDropTombstones = props.getProperties().get("transforms.unwrap.drop.tombstones");
|
||||
|
||||
@@ -153,15 +142,14 @@ public class DebeziumDeleteHandlingIntegrationTest {
|
||||
else if (deleteHandlingMode.equals("rewrite")) {
|
||||
received = outputDestination.receive(Duration.ofSeconds(10).toMillis(), DebeziumTestUtils.BINDING_NAME);
|
||||
assertThat(received).isNotNull();
|
||||
assertThat(toString(received.getPayload()).contains("\"__deleted\":\"true\""));
|
||||
assertThat(toString(received.getPayload())).contains("\"__deleted\":\"true\"");
|
||||
}
|
||||
|
||||
if (!(isDropTombstones.equals("true")) && isKafkaPresent) {
|
||||
if (!(isDropTombstones.equals("true"))) {
|
||||
received = outputDestination.receive(Duration.ofSeconds(10).toMillis(), DebeziumTestUtils.BINDING_NAME);
|
||||
assertThat(received).isNotNull();
|
||||
// Tombstones event should have KafkaNull payload
|
||||
assertThat(received.getPayload().getClass().getCanonicalName())
|
||||
.isEqualTo(DebeziumReactiveConsumerConfiguration.ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL);
|
||||
assertThat(received.getPayload()).isEqualTo(Optional.empty());
|
||||
|
||||
Object keyRaw = received.getHeaders().get("debezium_key");
|
||||
String key = (keyRaw instanceof byte[]) ? new String((byte[]) keyRaw) : "" + keyRaw;
|
||||
|
||||
@@ -18,6 +18,7 @@ package org.springframework.cloud.stream.app.source.debezium.integration;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import net.javacrumbs.jsonunit.JsonAssert;
|
||||
import net.javacrumbs.jsonunit.core.Configuration;
|
||||
@@ -27,19 +28,15 @@ import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
import org.testcontainers.junit.jupiter.Testcontainers;
|
||||
|
||||
import org.springframework.boot.test.context.FilteredClassLoader;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
import org.springframework.boot.test.context.runner.ContextConsumer;
|
||||
import org.springframework.cloud.fn.common.debezium.DebeziumProperties;
|
||||
import org.springframework.cloud.fn.supplier.debezium.DebeziumReactiveConsumerConfiguration;
|
||||
import org.springframework.cloud.stream.binder.test.OutputDestination;
|
||||
import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.kafka.support.KafkaNull;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.test.jdbc.JdbcTestUtils;
|
||||
import org.springframework.util.ClassUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
@@ -96,14 +93,7 @@ public class DebeziumFlatteningIntegrationTest {
|
||||
"app.datasource.type=com.zaxxer.hikari.HikariDataSource");
|
||||
|
||||
@Test
|
||||
public void noFlattenedResponseNoKafka() {
|
||||
contextRunner
|
||||
.withClassLoader(new FilteredClassLoader(KafkaNull.class)) // Remove Kafka from the classpath
|
||||
.run(noFlatteningTest);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void noFlattenedResponseWithKafka() {
|
||||
public void noFlattenedResponse() {
|
||||
contextRunner.run(noFlatteningTest);
|
||||
}
|
||||
|
||||
@@ -111,15 +101,11 @@ public class DebeziumFlatteningIntegrationTest {
|
||||
OutputDestination outputDestination = context.getBean(OutputDestination.class);
|
||||
JdbcTemplate jdbcTemplate = context.getBean(JdbcTemplate.class);
|
||||
|
||||
boolean isKafkaPresent = ClassUtils.isPresent(
|
||||
DebeziumReactiveConsumerConfiguration.ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL,
|
||||
context.getClassLoader());
|
||||
|
||||
List<Message<?>> messages = DebeziumTestUtils.receiveAll(outputDestination);
|
||||
assertThat(messages).hasSizeGreaterThanOrEqualTo(52);
|
||||
|
||||
JsonAssert.assertJsonEquals(DebeziumTestUtils.resourceToString(
|
||||
"classpath:/json/mysql_ddl_drop_inventory_address_table.json"),
|
||||
"classpath:/json/mysql_ddl_drop_inventory_address_table.json"),
|
||||
toString(messages.get(1).getPayload()),
|
||||
Configuration.empty().whenIgnoringPaths("schemaName", "tableChanges", "source.sequence",
|
||||
"source.ts_ms", "ts_ms"));
|
||||
@@ -143,7 +129,7 @@ public class DebeziumFlatteningIntegrationTest {
|
||||
|
||||
messages = DebeziumTestUtils.receiveAll(outputDestination);
|
||||
|
||||
assertThat(messages).hasSize(isKafkaPresent ? 4 : 3);
|
||||
assertThat(messages).hasSize(4);
|
||||
|
||||
JsonAssert.assertJsonEquals(
|
||||
DebeziumTestUtils.resourceToString("classpath:/json/mysql_update_inventory_customers.json"),
|
||||
@@ -160,30 +146,13 @@ public class DebeziumFlatteningIntegrationTest {
|
||||
JsonAssert.assertJsonEquals("{\"id\":" + newRecordId + "}",
|
||||
toString(messages.get(1).getHeaders().get("debezium_key")));
|
||||
|
||||
if (isKafkaPresent) {
|
||||
assertThat(messages.get(3).getPayload().getClass().getCanonicalName())
|
||||
.isEqualTo(DebeziumReactiveConsumerConfiguration.ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL,
|
||||
"Tombstones event should have KafkaNull payload");
|
||||
assertThat(messages.get(3).getHeaders().get("debezium_destination"))
|
||||
.isEqualTo("my-topic.inventory.customers");
|
||||
JsonAssert.assertJsonEquals("{\"id\":" + newRecordId + "}",
|
||||
toString(messages.get(3).getHeaders().get("debezium_key")));
|
||||
}
|
||||
assertThat(messages.get(3).getPayload()).isEqualTo(Optional.empty());
|
||||
assertThat(messages.get(3).getHeaders().get("debezium_destination"))
|
||||
.isEqualTo("my-topic.inventory.customers");
|
||||
JsonAssert.assertJsonEquals("{\"id\":" + newRecordId + "}",
|
||||
toString(messages.get(3).getHeaders().get("debezium_key")));
|
||||
};
|
||||
|
||||
@Test
|
||||
public void flattenedResponseNoKafka() {
|
||||
contextRunner
|
||||
.withPropertyValues("debezium.properties.transforms=unwrap",
|
||||
"debezium.properties.transforms.unwrap.type=io.debezium.transforms.ExtractNewRecordState",
|
||||
"debezium.properties.transforms.unwrap.add.fields=name,db,op",
|
||||
"debezium.properties.transforms.unwrap.add.headers=name,op",
|
||||
"debezium.properties.transforms.unwrap.delete.handling.mode=none",
|
||||
"debezium.properties.transforms.unwrap.drop.tombstones=false")
|
||||
.withClassLoader(new FilteredClassLoader(KafkaNull.class)) // Remove Kafka from the classpath
|
||||
.run(flatteningTest);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void flattenedResponseWithKafka() {
|
||||
contextRunner
|
||||
@@ -212,10 +181,6 @@ public class DebeziumFlatteningIntegrationTest {
|
||||
OutputDestination outputDestination = context.getBean(OutputDestination.class);
|
||||
JdbcTemplate jdbcTemplate = context.getBean(JdbcTemplate.class);
|
||||
|
||||
boolean isKafkaPresent = ClassUtils.isPresent(
|
||||
DebeziumReactiveConsumerConfiguration.ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL,
|
||||
context.getClassLoader());
|
||||
|
||||
List<Message<?>> messages = DebeziumTestUtils.receiveAll(outputDestination);
|
||||
assertThat(messages).hasSizeGreaterThanOrEqualTo(52);
|
||||
|
||||
@@ -225,7 +190,7 @@ public class DebeziumFlatteningIntegrationTest {
|
||||
String isDropTombstones = props.getProperties().get("transforms.unwrap.drop.tombstones");
|
||||
|
||||
JsonAssert.assertJsonEquals(DebeziumTestUtils.resourceToString(
|
||||
"classpath:/json/mysql_ddl_drop_inventory_address_table.json"),
|
||||
"classpath:/json/mysql_ddl_drop_inventory_address_table.json"),
|
||||
toString(messages.get(1).getPayload()),
|
||||
Configuration.empty().whenIgnoringPaths("schemaName", "tableChanges", "source.sequence",
|
||||
"source.ts_ms", "ts_ms"));
|
||||
@@ -256,7 +221,7 @@ public class DebeziumFlatteningIntegrationTest {
|
||||
|
||||
messages = DebeziumTestUtils.receiveAll(outputDestination);
|
||||
|
||||
assertThat(messages).hasSize((isDropTombstones.equals("false") && isKafkaPresent) ? 4 : 3);
|
||||
assertThat(messages).hasSize((isDropTombstones.equals("false")) ? 4 : 3);
|
||||
|
||||
JsonAssert.assertJsonEquals(
|
||||
DebeziumTestUtils.resourceToString("classpath:/json/mysql_flattened_update_inventory_customers.json"),
|
||||
@@ -273,10 +238,8 @@ public class DebeziumFlatteningIntegrationTest {
|
||||
toString(messages.get(1).getHeaders().get("debezium_key")));
|
||||
}
|
||||
|
||||
if (isDropTombstones.equals("false") && isKafkaPresent) {
|
||||
assertThat(messages.get(3).getPayload().getClass().getCanonicalName())
|
||||
.isEqualTo(DebeziumReactiveConsumerConfiguration.ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL,
|
||||
"Tombstones event should have KafkaNull payload");
|
||||
if (isDropTombstones.equals("false")) {
|
||||
assertThat(messages.get(3).getPayload()).isEqualTo(Optional.empty());
|
||||
assertThat(messages.get(3).getHeaders().get("debezium_destination"))
|
||||
.isEqualTo("my-topic.inventory.customers");
|
||||
JsonAssert.assertJsonEquals("{\"id\":" + newRecordId + "}",
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>file-supplier</artifactId>
|
||||
<artifactId>spring-file-supplier</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.stream.app</groupId>
|
||||
@@ -49,13 +49,13 @@
|
||||
<name>file</name>
|
||||
<type>source</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.supplier.file.FileSupplierConfiguration.class</configClass>
|
||||
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>fileSupplier</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>file-supplier</artifactId>
|
||||
<artifactId>spring-file-supplier</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.stream.app</groupId>
|
||||
|
||||
@@ -17,11 +17,17 @@
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>ftp-supplier</artifactId>
|
||||
<artifactId>spring-ftp-supplier</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>function-test-support</artifactId>
|
||||
<artifactId>spring-function-test-support</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.ftpserver</groupId>
|
||||
<artifactId>ftpserver-core</artifactId>
|
||||
<version>1.2.0</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
@@ -54,13 +60,13 @@
|
||||
<name>ftp</name>
|
||||
<type>source</type>
|
||||
<version>${project.version}</version>
|
||||
<configClass>org.springframework.cloud.fn.supplier.ftp.FtpSupplierConfiguration.class</configClass>
|
||||
|
||||
<configClass>AUTOCONFIGURATION</configClass>
|
||||
<functionDefinition>ftpSupplier</functionDefinition>
|
||||
<maven>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.fn</groupId>
|
||||
<artifactId>ftp-supplier</artifactId>
|
||||
<artifactId>spring-ftp-supplier</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud.stream.app</groupId>
|
||||
|
||||
@@ -23,7 +23,6 @@ import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.cloud.fn.supplier.ftp.FtpSupplierConfiguration;
|
||||
import org.springframework.cloud.fn.supplier.ftp.FtpSupplierProperties;
|
||||
import org.springframework.cloud.fn.test.support.ftp.FtpTestSupport;
|
||||
import org.springframework.cloud.stream.binder.test.OutputDestination;
|
||||
@@ -54,7 +53,7 @@ public class FtpSourceTests extends FtpTestSupport {
|
||||
FtpSupplierProperties config;
|
||||
|
||||
@Test
|
||||
public void testFtpource() {
|
||||
public void testFtpSource() {
|
||||
Message<byte[]> message = output.receive(10000, "ftpSupplier-out-0");
|
||||
assertThat(new File(new String(message.getPayload()).replaceAll("\"", ""))).isEqualTo(
|
||||
new File(this.config.getLocalDir(), "ftpSource1.txt"));
|
||||
@@ -64,7 +63,7 @@ public class FtpSourceTests extends FtpTestSupport {
|
||||
}
|
||||
|
||||
@SpringBootApplication
|
||||
@Import({TestChannelBinderConfiguration.class, FtpSupplierConfiguration.class})
|
||||
@Import(TestChannelBinderConfiguration.class)
|
||||
public static class SampleConfiguration {
|
||||
|
||||
@Bean
|
||||
@@ -77,4 +76,5 @@ public class FtpSourceTests extends FtpTestSupport {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user