SpringCloud/spring-functions-catalog
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-7: Add checkstyle and javaformat plugins

Browse Source 
Fixes: #7

* Run `./gradlew format`
* Updates from PR review suggestions
This commit is contained in:
Chris Bono
2023-12-29 21:08:58 -06:00
committed by Artem Bilan
parent 84e732da08
commit 836708f0f2
357 changed files with 4344 additions and 4519 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
build.gradle
View File

@@ -1,10 +1,12 @@
buildscript {
ext.isCI = System.getenv('GITHUB_ACTION')
ext.javaFormatVersion = '0.0.40'
}
plugins {
id 'base'
id 'io.spring.dependency-management' version '1.1.4'
id "io.spring.javaformat" version "${javaFormatVersion}" apply false
id 'com.github.spotbugs' version '6.0.4'
id 'com.google.protobuf' version '0.9.4' apply false
}
@@ -58,12 +60,14 @@ allprojects {
}
configure(javaProjects) { subproject ->
apply plugin: 'java-library'
apply plugin: 'eclipse'
apply plugin: 'idea'
apply from: "${rootDir}/publish-maven.gradle"
apply from: "${rootDir}/gradle/checkstyle-conventions.gradle"
sourceSets {
test {

2
common/spring-aws-s3-common/src/main/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3Configuration.java
View File

@@ -33,7 +33,7 @@ import org.springframework.integration.aws.support.S3SessionFactory;
* @author Artem Bilan
*/
@AutoConfiguration
@AutoConfigureAfter({S3AutoConfiguration.class, S3CrtAsyncClientAutoConfiguration.class})
@AutoConfigureAfter({ S3AutoConfiguration.class, S3CrtAsyncClientAutoConfiguration.class })
public class AmazonS3Configuration {
@Bean

25
common/spring-aws-s3-common/src/test/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3ConfigurationTests.java
View File

@@ -43,13 +43,9 @@ import org.springframework.integration.test.util.TestUtils;
public class AmazonS3ConfigurationTests {
private final ApplicationContextRunner runner = new ApplicationContextRunner()
.withConfiguration(
AutoConfigurations.of(
AwsAutoConfiguration.class,
S3AutoConfiguration.class,
S3CrtAsyncClientAutoConfiguration.class,
AmazonS3Configuration.class))
.withUserConfiguration(TestConfiguration.class);
.withConfiguration(AutoConfigurations.of(AwsAutoConfiguration.class, S3AutoConfiguration.class,
S3CrtAsyncClientAutoConfiguration.class, AmazonS3Configuration.class))
.withUserConfiguration(TestConfiguration.class);
private static final String TEST_REGION_NAME = "eu-central-1";
@@ -61,22 +57,21 @@ public class AmazonS3ConfigurationTests {
S3Utilities utilities = amazonS3.utilities();
Assertions.assertEquals(TEST_REGION_NAME,
TestUtils.getPropertyValue(utilities, "region", Region.class).id());
Assertions.assertTrue(
utilities.getUrl(GetUrlRequest.builder().bucket("b").key("k").build()).toString()
.startsWith("https://s3.eu-central-1.amazonaws.com"));
Assertions.assertTrue(utilities.getUrl(GetUrlRequest.builder().bucket("b").key("k").build())
.toString()
.startsWith("https://s3.eu-central-1.amazonaws.com"));
});
}
@Test
public void testAmazonS3ConfigurationForS3CompatibleStorage() {
runner.withPropertyValues(
"spring.cloud.aws.s3.endpoint=http://localhost:8080"
).run(context -> {
runner.withPropertyValues("spring.cloud.aws.s3.endpoint=http://localhost:8080").run(context -> {
S3Client amazonS3 = context.getBean(S3Client.class);
Assertions.assertNotNull(amazonS3);
S3Utilities utilities = amazonS3.utilities();
Assertions.assertTrue(utilities.getUrl(GetUrlRequest.builder().bucket("b").key("k").build()).toString()
.startsWith("http://localhost:8080"));
Assertions.assertTrue(utilities.getUrl(GetUrlRequest.builder().bucket("b").key("k").build())
.toString()
.startsWith("http://localhost:8080"));
});
}

6
common/spring-config-common/src/main/java/org/springframework/cloud/fn/common/config/ComponentCustomizer.java
View File

@@ -17,13 +17,11 @@
package org.springframework.cloud.fn.common.config;
/**
* The customizer contract to apply to beans in the application context which
* type is matching to generic type of the instance of this interface.
* The customizer contract to apply to beans in the application context which type is
* matching to generic type of the instance of this interface.
*
* @param <T> the target component (bean) type in the application context to customize.
*
* @author Artem Bilan
*
* @since 1.2.1
*/
@FunctionalInterface

8
common/spring-config-common/src/main/java/org/springframework/cloud/fn/common/config/SpelExpressionConverterConfiguration.java
View File

@@ -42,14 +42,15 @@ import org.springframework.integration.json.JsonPropertyAccessor;
public class SpelExpressionConverterConfiguration {
/**
* Specific Application Context name to be used as Bean qualifier when the {@link EvaluationContext} is injected.
* Specific Application Context name to be used as Bean qualifier when the
* {@link EvaluationContext} is injected.
*/
public static final String INTEGRATION_EVALUATION_CONTEXT = "integrationEvaluationContext";
@Bean
public static SpelPropertyAccessorRegistrar spelPropertyAccessorRegistrar() {
return (new SpelPropertyAccessorRegistrar())
.add(Introspector.decapitalize(JsonPropertyAccessor.class.getSimpleName()), new JsonPropertyAccessor());
.add(Introspector.decapitalize(JsonPropertyAccessor.class.getSimpleName()), new JsonPropertyAccessor());
}
@Bean
@@ -60,6 +61,7 @@ public class SpelExpressionConverterConfiguration {
}
public static class SpelConverter implements Converter<String, Expression> {
private SpelExpressionParser parser = new SpelExpressionParser();
@Autowired
@@ -84,5 +86,7 @@ public class SpelExpressionConverterConfiguration {
String.format("Could not convert '%s' into a SpEL expression", source), var3);
}
}
}
}

96
common/spring-debezium-autoconfigure/src/main/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfiguration.java
View File

@@ -46,16 +46,20 @@ import org.springframework.core.annotation.Order;
/**
* {@link EnableAutoConfiguration Auto-configuration} for {@link DebeziumEngine.Builder}.
* <p>
* The builder provides a standalone engine configuration that talks with the source data system.
* The builder provides a standalone engine configuration that talks with the source data
* system.
* <p>
* The application that runs the debezium engine assumes all responsibility for fault tolerance, scalability, and
* durability. Additionally, applications must specify how the engine can store its relational database schema history
* and offsets. By default, this information will be stored in memory and will thus be lost upon application restart.
* The application that runs the debezium engine assumes all responsibility for fault
* tolerance, scalability, and durability. Additionally, applications must specify how the
* engine can store its relational database schema history and offsets. By default, this
* information will be stored in memory and will thus be lost upon application restart.
* <p>
* The {@link DebeziumEngine.Builder} auto-configuration is activated only if a Debezium Connector is available on the
* classpath and the <code>debezium.properties.connector.class</code> property is set.
* The {@link DebeziumEngine.Builder} auto-configuration is activated only if a Debezium
* Connector is available on the classpath and the
* <code>debezium.properties.connector.class</code> property is set.
* <p>
* Properties prefixed with <code>debezium.properties</code> are passed through as native Debezium properties.
* Properties prefixed with <code>debezium.properties</code> are passed through as native
* Debezium properties.
*
* @author Christian Tzolov
* @author Corneil du Plessis
@@ -69,29 +73,31 @@ public class DebeziumEngineBuilderAutoConfiguration {
private static final Log logger = LogFactory.getLog(DebeziumEngineBuilderAutoConfiguration.class);
/**
* The fully-qualified class name of the commit policy type. The default is a periodic commit policy based upon time
* intervals.
* @param properties The 'debezium.properties.offset.flush.interval.ms' configuration is compulsory for the Periodic
* policy type. The ALWAYS and DEFAULT doesn't require additional configuration.
* The fully-qualified class name of the commit policy type. The default is a periodic
* commit policy based upon time intervals.
* @param properties The 'debezium.properties.offset.flush.interval.ms' configuration
* is compulsory for the Periodic policy type. The ALWAYS and DEFAULT doesn't require
* additional configuration.
*/
@Bean
@ConditionalOnMissingBean
public OffsetCommitPolicy offsetCommitPolicy(DebeziumProperties properties) {
switch (properties.getOffsetCommitPolicy()) {
case PERIODIC:
return OffsetCommitPolicy.periodic(properties.getDebeziumNativeConfiguration());
case ALWAYS:
return OffsetCommitPolicy.always();
case DEFAULT:
default:
return NULL_OFFSET_COMMIT_POLICY;
case PERIODIC:
return OffsetCommitPolicy.periodic(properties.getDebeziumNativeConfiguration());
case ALWAYS:
return OffsetCommitPolicy.always();
case DEFAULT:
default:
return NULL_OFFSET_COMMIT_POLICY;
}
}
/**
* Use the specified clock when needing to determine the current time. Defaults to {@link Clock#systemDefaultZone()
* system clock}, but you can override the Bean in your configuration with you {@link Clock implementation}. Returns
* Use the specified clock when needing to determine the current time. Defaults to
* {@link Clock#systemDefaultZone() system clock}, but you can override the Bean in
* your configuration with you {@link Clock implementation}. Returns
* @return Clock for the system default zone.
*/
@Bean
@@ -101,9 +107,10 @@ public class DebeziumEngineBuilderAutoConfiguration {
}
/**
* When the engine's {@link DebeziumEngine#run()} method completes, call the supplied function with the results.
* @return Default completion callback that logs the completion status. The bean can be overridden in custom
* implementation.
* When the engine's {@link DebeziumEngine#run()} method completes, call the supplied
* function with the results.
* @return Default completion callback that logs the completion status. The bean can
* be overridden in custom implementation.
*/
@Bean
@ConditionalOnMissingBean
@@ -112,8 +119,9 @@ public class DebeziumEngineBuilderAutoConfiguration {
}
/**
* During the engine run, provides feedback about the different stages according to the completion state of each
* component running within the engine (connectors, tasks etc). The bean can be overridden in custom implementation.
* During the engine run, provides feedback about the different stages according to
* the completion state of each component running within the engine (connectors, tasks
* etc). The bean can be overridden in custom implementation.
*/
@Bean
@ConditionalOnMissingBean
@@ -135,29 +143,29 @@ public class DebeziumEngineBuilderAutoConfiguration {
serializationFormatClass(properties.getHeaderFormat()),
"Cannot find header format for " + properties.getProperties());
return DebeziumEngine
.create(KeyValueHeaderChangeEventFormat.of(payloadFormat, payloadFormat, headerFormat))
.using(properties.getDebeziumNativeConfiguration())
.using(debeziumClock)
.using(completionCallback)
.using(connectorCallback)
.using((offsetCommitPolicy != NULL_OFFSET_COMMIT_POLICY) ? offsetCommitPolicy : null);
return DebeziumEngine.create(KeyValueHeaderChangeEventFormat.of(payloadFormat, payloadFormat, headerFormat))
.using(properties.getDebeziumNativeConfiguration())
.using(debeziumClock)
.using(completionCallback)
.using(connectorCallback)
.using((offsetCommitPolicy != NULL_OFFSET_COMMIT_POLICY) ? offsetCommitPolicy : null);
}
/**
* Converts the {@link DebeziumFormat} enum into Debezium {@link SerializationFormat} class.
* Converts the {@link DebeziumFormat} enum into Debezium {@link SerializationFormat}
* class.
* @param debeziumFormat debezium format property.
*/
private Class<? extends SerializationFormat<byte[]>> serializationFormatClass(DebeziumFormat debeziumFormat) {
switch (debeziumFormat) {
case JSON:
return io.debezium.engine.format.JsonByteArray.class;
case AVRO:
return io.debezium.engine.format.Avro.class;
case PROTOBUF:
return io.debezium.engine.format.Protobuf.class;
default:
throw new IllegalArgumentException("Unknown debezium format: " + debeziumFormat);
case JSON:
return io.debezium.engine.format.JsonByteArray.class;
case AVRO:
return io.debezium.engine.format.Avro.class;
case PROTOBUF:
return io.debezium.engine.format.Protobuf.class;
default:
throw new IllegalArgumentException("Unknown debezium format: " + debeziumFormat);
}
}
@@ -173,7 +181,8 @@ public class DebeziumEngineBuilderAutoConfiguration {
};
/**
* Callback function which informs users about the various stages a connector goes through during startup.
* Callback function which informs users about the various stages a connector goes
* through during startup.
*/
private static final ConnectorCallback DEFAULT_CONNECTOR_CALLBACK = new ConnectorCallback() {
@@ -218,7 +227,8 @@ public class DebeziumEngineBuilderAutoConfiguration {
};
/**
* Determine if Debezium connector is available. This either kicks in if any debezium connector is available.
* Determine if Debezium connector is available. This either kicks in if any debezium
* connector is available.
*/
@Order(Ordered.LOWEST_PRECEDENCE)
static class OnDebeziumConnectorCondition extends AnyNestedCondition {

23
common/spring-debezium-autoconfigure/src/main/java/org/springframework/cloud/fn/common/debezium/DebeziumProperties.java
View File

@@ -29,6 +29,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
public class DebeziumProperties {
public enum DebeziumFormat {
/**
* JSON change event format.
*/
@@ -51,11 +52,12 @@ public class DebeziumProperties {
public final String contentType() {
return contentType;
}
};
/**
* Spring pass-trough wrapper for debezium configuration properties. All properties with a 'debezium.properties.*'
* prefix are native Debezium properties.
* Spring pass-trough wrapper for debezium configuration properties. All properties
* with a 'debezium.properties.*' prefix are native Debezium properties.
*/
private Map<String, String> properties = new HashMap<>();
@@ -95,21 +97,24 @@ public class DebeziumProperties {
}
public enum DebeziumOffsetCommitPolicy {
/**
* Commits offsets as frequently as possible. This may result in reduced performance, but it has the least
* potential for seeing source records more than once upon restart.
* Commits offsets as frequently as possible. This may result in reduced
* performance, but it has the least potential for seeing source records more than
* once upon restart.
*/
ALWAYS,
/**
* Commits offsets no more than the specified time period. If the specified time is less than {@code 0} then the
* policy will behave as ALWAYS policy. Requires the 'debezium.properties.offset.flush.interval.ms' native
* property to be set.
* Commits offsets no more than the specified time period. If the specified time
* is less than {@code 0} then the policy will behave as ALWAYS policy. Requires
* the 'debezium.properties.offset.flush.interval.ms' native property to be set.
*/
PERIODIC,
/**
* Uses the default Debezium engine policy (PERIODIC).
*/
DEFAULT;
}
public DebeziumOffsetCommitPolicy getOffsetCommitPolicy() {
@@ -121,11 +126,13 @@ public class DebeziumProperties {
}
/**
* Converts the Spring Framework "debezium.properties.*" properties into native Debezium configuration.
* Converts the Spring Framework "debezium.properties.*" properties into native
* Debezium configuration.
*/
public Properties getDebeziumNativeConfiguration() {
Properties outProps = new java.util.Properties();
outProps.putAll(this.getProperties());
return outProps;
}
}

17
common/spring-debezium-autoconfigure/src/main/java/org/springframework/cloud/fn/common/debezium/EmbeddedEngineExecutorService.java
View File

@@ -28,14 +28,16 @@ import org.apache.commons.logging.LogFactory;
import org.springframework.context.SmartLifecycle;
/**
* The Debezium Engine is designed to be submitted to an {@link Executor} or {@link ExecutorService} for execution by a
* single thread, and a running connector can be stopped either by calling {@link #stop()} from another thread or by
* interrupting the running thread (e.g., as is the case with {@link ExecutorService#shutdownNow()}).
* The Debezium Engine is designed to be submitted to an {@link Executor} or
* {@link ExecutorService} for execution by a single thread, and a running connector can
* be stopped either by calling {@link #stop()} from another thread or by interrupting the
* running thread (e.g., as is the case with {@link ExecutorService#shutdownNow()}).
*
* The EmbeddedEngineExecutorService provides a sample ExecutorService implementation aligned with the Spring lifecycle.
* The EmbeddedEngineExecutorService provides a sample ExecutorService implementation
* aligned with the Spring lifecycle.
*
* Note that the DebeziumReactiveConsumerConfiguration embeds an ExecutorService as part of the
* Supplier&lt;Flux&lt;Message&lt;?&gt;&gt;&gt; configuration.
* Note that the DebeziumReactiveConsumerConfiguration embeds an ExecutorService as part
* of the Supplier&lt;Flux&lt;Message&lt;?&gt;&gt;&gt; configuration.
*
* @author Christian Tzolov
*/
@@ -44,7 +46,9 @@ public class EmbeddedEngineExecutorService implements SmartLifecycle, AutoClosea
private static final Log logger = LogFactory.getLog(EmbeddedEngineExecutorService.class);
private final DebeziumEngine<?> engine;
private final ExecutorService executor;
private final AtomicBoolean running = new AtomicBoolean(false);
public EmbeddedEngineExecutorService(DebeziumEngine<?> engine) {
@@ -81,4 +85,5 @@ public class EmbeddedEngineExecutorService implements SmartLifecycle, AutoClosea
public boolean isRunning() {
return this.running.get();
}
}

105
common/spring-debezium-autoconfigure/src/test/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfigurationIntegrationTest.java
View File

@@ -54,18 +54,21 @@ import org.springframework.test.jdbc.JdbcTestUtils;
import static org.awaitility.Awaitility.await;
/**
* This test illustrate how to leverage the DebeziumEngineAutoConfiguration to build a consumer function with a custom
* change event Consumer.
* This test illustrate how to leverage the DebeziumEngineAutoConfiguration to build a
* consumer function with a custom change event Consumer.
*
* @author Christian Tzolov
*/
@Tag("integration")
@Testcontainers
public class DebeziumEngineBuilderAutoConfigurationIntegrationTest {
private static final Log logger = LogFactory.getLog(DebeziumEngineBuilderAutoConfigurationIntegrationTest.class);
private static final String DATABASE_NAME = "inventory";
public static final String IMAGE_TAG = "2.3.0.Final";
public static final String DEBEZIUM_EXAMPLE_MYSQL_IMAGE = "debezium/example-mysql:" + IMAGE_TAG;
@TempDir
@@ -73,70 +76,64 @@ public class DebeziumEngineBuilderAutoConfigurationIntegrationTest {
@Container
static GenericContainer<?> debeziumMySQL = new GenericContainer<>(DEBEZIUM_EXAMPLE_MYSQL_IMAGE)
.withEnv("MYSQL_ROOT_PASSWORD", "debezium")
.withEnv("MYSQL_USER", "mysqluser")
.withEnv("MYSQL_PASSWORD", "mysqlpw")
.withExposedPorts(3306);
.withEnv("MYSQL_ROOT_PASSWORD", "debezium")
.withEnv("MYSQL_USER", "mysqluser")
.withEnv("MYSQL_PASSWORD", "mysqlpw")
.withExposedPorts(3306);
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withUserConfiguration(DebeziumCustomConsumerApplication.class)
.withPropertyValues(
"spring.datasource.type=com.zaxxer.hikari.HikariDataSource",
.withUserConfiguration(DebeziumCustomConsumerApplication.class)
.withPropertyValues("spring.datasource.type=com.zaxxer.hikari.HikariDataSource",
"debezium.properties.offset.storage=org.apache.kafka.connect.storage.FileOffsetBackingStore",
"debezium.properties.offset.storage.file.filename=" + anotherTempDir.getAbsolutePath()
+ "offsets.dat",
"debezium.properties.offset.flush.interval.ms=60000",
"debezium.properties.offset.storage=org.apache.kafka.connect.storage.FileOffsetBackingStore",
"debezium.properties.offset.storage.file.filename=" + anotherTempDir.getAbsolutePath() + "offsets.dat",
"debezium.properties.offset.flush.interval.ms=60000",
"debezium.properties.schema.history.internal=io.debezium.storage.file.history.FileSchemaHistory", // new
"debezium.properties.schema.history.internal.file.filename=" + anotherTempDir.getAbsolutePath()
+ "schemahistory.dat",
"debezium.properties.schema.history.internal=io.debezium.storage.file.history.FileSchemaHistory", // new
"debezium.properties.schema.history.internal.file.filename=" + anotherTempDir.getAbsolutePath()
+ "schemahistory.dat",
"debezium.properties.topic.prefix=my-topic",
"debezium.properties.topic.prefix=my-topic",
"debezium.properties.name=my-sql-connector",
"debezium.properties.connector.class=io.debezium.connector.mysql.MySqlConnector",
"debezium.properties.name=my-sql-connector",
"debezium.properties.connector.class=io.debezium.connector.mysql.MySqlConnector",
"debezium.properties.database.user=debezium",
"debezium.properties.database.password=dbz",
"debezium.properties.database.hostname=localhost",
"debezium.properties.database.port=" + debeziumMySQL.getMappedPort(3306),
"debezium.properties.database.server.id=85744",
"debezium.properties.database.user=debezium", "debezium.properties.database.password=dbz",
"debezium.properties.database.hostname=localhost",
"debezium.properties.database.port=" + debeziumMySQL.getMappedPort(3306),
"debezium.properties.database.server.id=85744",
// JdbcTemplate configuration
String.format("app.datasource.url=jdbc:mysql://localhost:%d/%s?enabledTLSProtocols=TLSv1.2",
debeziumMySQL.getMappedPort(3306), DATABASE_NAME),
"app.datasource.username=root",
"app.datasource.password=debezium",
"app.datasource.driver-class-name=com.mysql.cj.jdbc.Driver",
"app.datasource.type=com.zaxxer.hikari.HikariDataSource");
// JdbcTemplate configuration
String.format("app.datasource.url=jdbc:mysql://localhost:%d/%s?enabledTLSProtocols=TLSv1.2",
debeziumMySQL.getMappedPort(3306), DATABASE_NAME),
"app.datasource.username=root", "app.datasource.password=debezium",
"app.datasource.driver-class-name=com.mysql.cj.jdbc.Driver",
"app.datasource.type=com.zaxxer.hikari.HikariDataSource");
@Test
public void consumerTest() {
logger.info("Temp dir: " + anotherTempDir.getAbsolutePath());
contextRunner
.withPropertyValues(
// Flattering:
// https://debezium.io/documentation/reference/stable/transformations/event-flattening.html
"debezium.properties.transforms=unwrap",
"debezium.properties.transforms.unwrap.type=io.debezium.transforms.ExtractNewRecordState",
"debezium.properties.transforms.unwrap.drop.tombstones=false",
"debezium.properties.transforms.unwrap.delete.handling.mode=rewrite",
"debezium.properties.transforms.unwrap.add.fields=name,db")
.run(context -> {
JdbcTemplate jdbcTemplate = context.getBean(JdbcTemplate.class);
contextRunner.withPropertyValues(
// Flattering:
// https://debezium.io/documentation/reference/stable/transformations/event-flattening.html
"debezium.properties.transforms=unwrap",
"debezium.properties.transforms.unwrap.type=io.debezium.transforms.ExtractNewRecordState",
"debezium.properties.transforms.unwrap.drop.tombstones=false",
"debezium.properties.transforms.unwrap.delete.handling.mode=rewrite",
"debezium.properties.transforms.unwrap.add.fields=name,db")
.run(context -> {
JdbcTemplate jdbcTemplate = context.getBean(JdbcTemplate.class);
DebeziumCustomConsumerApplication.TestDebeziumConsumer testConsumer = context
.getBean(DebeziumCustomConsumerApplication.TestDebeziumConsumer.class);
jdbcTemplate.update(
"insert into `customers`(`first_name`,`last_name`,`email`) " +
"VALUES('Test666', 'Test666', 'Test666@spring.org')");
JdbcTestUtils.deleteFromTableWhere(jdbcTemplate, "customers", "first_name = ?", "Test666");
DebeziumCustomConsumerApplication.TestDebeziumConsumer testConsumer = context
.getBean(DebeziumCustomConsumerApplication.TestDebeziumConsumer.class);
jdbcTemplate.update("insert into `customers`(`first_name`,`last_name`,`email`) "
+ "VALUES('Test666', 'Test666', 'Test666@spring.org')");
JdbcTestUtils.deleteFromTableWhere(jdbcTemplate, "customers", "first_name = ?", "Test666");
await().atMost(Duration.ofSeconds(30)).until(() -> (testConsumer.recordList.size() >= 52));
});
await().atMost(Duration.ofSeconds(30)).until(() -> (testConsumer.recordList.size() >= 52));
});
}
@SpringBootConfiguration
@@ -157,16 +154,14 @@ public class DebeziumEngineBuilderAutoConfigurationIntegrationTest {
@Bean
public HikariDataSource dataSource(DataSourceProperties dataSourceProperties) {
return dataSourceProperties.initializeDataSourceBuilder()
.type(HikariDataSource.class)
.build();
return dataSourceProperties.initializeDataSourceBuilder().type(HikariDataSource.class).build();
}
@Bean
public EmbeddedEngineExecutorService embeddedEngine(Consumer<ChangeEvent<byte[], byte[]>> changeEventConsumer,
Builder<ChangeEvent<byte[], byte[]>> debeziumEngineBuilder) {
DebeziumEngine<ChangeEvent<byte[], byte[]>> b = debeziumEngineBuilder.notifying(changeEventConsumer)
.build();
.build();
return new EmbeddedEngineExecutorService(debeziumEngineBuilder.notifying(changeEventConsumer).build());
}
@@ -196,7 +191,9 @@ public class DebeziumEngineBuilderAutoConfigurationIntegrationTest {
System.out.println("[Debezium Event]: " + changeEvent.toString());
}
}
}
}
}

10
common/spring-debezium-autoconfigure/src/test/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfigurationTests.java
View File

@@ -33,7 +33,7 @@ import static org.assertj.core.api.Assertions.assertThat;
public class DebeziumEngineBuilderAutoConfigurationTests {
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(DebeziumEngineBuilderAutoConfiguration.class));
.withConfiguration(AutoConfigurations.of(DebeziumEngineBuilderAutoConfiguration.class));
// We have the debezium connectors on the classpath by default.
@@ -47,10 +47,10 @@ public class DebeziumEngineBuilderAutoConfigurationTests {
@Test
void noConnectorWithProperty() {
this.contextRunner.withPropertyValues("debezium.properties.connector.class=Dummy")
.withClassLoader(new FilteredClassLoader("io.debezium.connector"))
.run((context) -> {
assertThat(context).doesNotHaveBean(DebeziumEngine.Builder.class);
});
.withClassLoader(new FilteredClassLoader("io.debezium.connector"))
.run((context) -> {
assertThat(context).doesNotHaveBean(DebeziumEngine.Builder.class);
});
}
@Test

18
common/spring-file-common/src/main/java/org/springframework/cloud/fn/common/file/FileConsumerProperties.java
View File

@@ -23,8 +23,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.validation.annotation.Validated;
/**
*
* @author David Turanski
* @author David Turanski
* @author Artem Bilan
*/
@ConfigurationProperties("file.consumer")
@@ -32,22 +31,20 @@ import org.springframework.validation.annotation.Validated;
public class FileConsumerProperties {
/**
* The FileReadingMode to use for file reading sources.
* Values are 'ref' - The File object,
* 'lines' - a message per line, or
* 'contents' - the contents as bytes.
* The FileReadingMode to use for file reading sources. Values are 'ref' - The File
* object, 'lines' - a message per line, or 'contents' - the contents as bytes.
*/
private FileReadingMode mode = FileReadingMode.contents;
/**
* Set to true to emit start of file/end of file marker messages before/after the data.
* Only valid with FileReadingMode 'lines'.
* Set to true to emit start of file/end of file marker messages before/after the
* data. Only valid with FileReadingMode 'lines'.
*/
private Boolean withMarkers = null;
/**
* When 'fileMarkers == true', specify if they should be produced
* as FileSplitter.FileMarker objects or JSON.
* When 'fileMarkers == true', specify if they should be produced as
* FileSplitter.FileMarker objects or JSON.
*/
private boolean markersJson = true;
@@ -80,4 +77,5 @@ public class FileConsumerProperties {
public boolean isWithMarkersValid() {
return this.withMarkers == null || FileReadingMode.lines == this.mode;
}
}

2
common/spring-file-common/src/main/java/org/springframework/cloud/fn/common/file/FileReadingMode.java
View File

@@ -23,6 +23,7 @@ package org.springframework.cloud.fn.common.file;
* @author David Turanski
*/
public enum FileReadingMode {
/**
* ref mode.
*/
@@ -35,4 +36,5 @@ public enum FileReadingMode {
* contents mode.
*/
contents;
}

36
common/spring-file-common/src/main/java/org/springframework/cloud/fn/common/file/FileUtils.java
View File

@@ -39,35 +39,35 @@ public final class FileUtils {
/**
* Enhance an {@link IntegrationFlowBuilder} to add flow snippets, depending on
* {@link FileConsumerProperties}.
*
* @param flowBuilder the flow builder.
* @param flowBuilder the flow builder.
* @param fileConsumerProperties the properties.
* @return the updated flow builder.
*/
public static IntegrationFlowBuilder enhanceFlowForReadingMode(IntegrationFlowBuilder flowBuilder,
FileConsumerProperties fileConsumerProperties) {
FileConsumerProperties fileConsumerProperties) {
switch (fileConsumerProperties.getMode()) {
case contents:
flowBuilder.enrichHeaders(Collections.<String, Object>singletonMap(MessageHeaders.CONTENT_TYPE,
MimeTypeUtils.APPLICATION_OCTET_STREAM_VALUE))
.transform(new FileToByteArrayTransformer());
.transform(new FileToByteArrayTransformer());
break;
case lines:
Boolean withMarkers = fileConsumerProperties.getWithMarkers();
if (withMarkers == null) {
withMarkers = false;
}
flowBuilder.enrichHeaders(Collections.<String, Object>singletonMap(MessageHeaders.CONTENT_TYPE,
MimeTypeUtils.TEXT_PLAIN_VALUE))
.split(new FileSplitter(true, withMarkers, fileConsumerProperties.getMarkersJson()));
flowBuilder
.enrichHeaders(Collections.<String, Object>singletonMap(MessageHeaders.CONTENT_TYPE,
MimeTypeUtils.TEXT_PLAIN_VALUE))
.split(new FileSplitter(true, withMarkers, fileConsumerProperties.getMarkersJson()));
break;
case ref:
flowBuilder.enrichHeaders(Collections.<String, Object>singletonMap(MessageHeaders.CONTENT_TYPE,
MimeTypeUtils.APPLICATION_JSON_VALUE));
break;
default:
throw new IllegalArgumentException(fileConsumerProperties.getMode().name() +
" is not a supported file reading mode.");
throw new IllegalArgumentException(
fileConsumerProperties.getMode().name() + " is not a supported file reading mode.");
}
return flowBuilder;
}
@@ -75,32 +75,32 @@ public final class FileUtils {
/**
* Enhance an {@link IntegrationFlowBuilder} to add flow snippets, depending on
* {@link FileConsumerProperties}; used for streaming sources.
*
* @param flowBuilder the flow builder.
* @param flowBuilder the flow builder.
* @param fileConsumerProperties the properties.
* @return the updated flow builder.
*/
public static IntegrationFlowBuilder enhanceStreamFlowForReadingMode(IntegrationFlowBuilder flowBuilder,
FileConsumerProperties fileConsumerProperties) {
FileConsumerProperties fileConsumerProperties) {
switch (fileConsumerProperties.getMode()) {
case contents:
flowBuilder.enrichHeaders(Collections.<String, Object>singletonMap(MessageHeaders.CONTENT_TYPE,
MimeTypeUtils.APPLICATION_OCTET_STREAM_VALUE))
.transform(new StreamTransformer());
.transform(new StreamTransformer());
break;
case lines:
Boolean withMarkers = fileConsumerProperties.getWithMarkers();
if (withMarkers == null) {
withMarkers = false;
}
flowBuilder.enrichHeaders(Collections.<String, Object>singletonMap(MessageHeaders.CONTENT_TYPE,
MimeTypeUtils.TEXT_PLAIN_VALUE))
.split(new FileSplitter(true, withMarkers, fileConsumerProperties.getMarkersJson()));
flowBuilder
.enrichHeaders(Collections.<String, Object>singletonMap(MessageHeaders.CONTENT_TYPE,
MimeTypeUtils.TEXT_PLAIN_VALUE))
.split(new FileSplitter(true, withMarkers, fileConsumerProperties.getMarkersJson()));
break;
case ref:
default:
throw new IllegalArgumentException(fileConsumerProperties.getMode().name() +
" is not a supported file reading mode when streaming.");
throw new IllegalArgumentException(fileConsumerProperties.getMode().name()
+ " is not a supported file reading mode when streaming.");
}
return flowBuilder;
}

4
common/spring-file-common/src/main/java/org/springframework/cloud/fn/common/file/remote/RemoteFileDeletingAdvice.java
View File

@@ -41,8 +41,7 @@ public class RemoteFileDeletingAdvice implements MessageSourceMutator {
* @param template the template.
* @param remoteFileSeparator the separator.
*/
public RemoteFileDeletingAdvice(RemoteFileTemplate<?> template,
String remoteFileSeparator) {
public RemoteFileDeletingAdvice(RemoteFileTemplate<?> template, String remoteFileSeparator) {
this.template = template;
this.remoteFileSeparator = remoteFileSeparator;
}
@@ -57,4 +56,5 @@ public class RemoteFileDeletingAdvice implements MessageSourceMutator {
}
return result;
}
}

6
common/spring-file-common/src/main/java/org/springframework/cloud/fn/common/file/remote/RemoteFileRenamingAdvice.java
View File

@@ -24,7 +24,6 @@ import org.springframework.integration.file.remote.RemoteFileTemplate;
import org.springframework.lang.Nullable;
import org.springframework.messaging.Message;
/**
* A {@link MessageSourceMutator} that renames a remote file on success.
*
@@ -45,9 +44,7 @@ public class RemoteFileRenamingAdvice implements MessageSourceMutator {
* @param remoteFileSeparator the separator.
* @param newNameExp the SpEl expression for the new name.
*/
public RemoteFileRenamingAdvice(RemoteFileTemplate<?> template,
String remoteFileSeparator,
Expression newNameExp) {
public RemoteFileRenamingAdvice(RemoteFileTemplate<?> template, String remoteFileSeparator, Expression newNameExp) {
this.template = template;
this.remoteFileSeparator = remoteFileSeparator;
this.newName = newNameExp;
@@ -66,4 +63,5 @@ public class RemoteFileRenamingAdvice implements MessageSourceMutator {
}
return result;
}
}

1
common/spring-ftp-common/src/main/java/org/springframework/cloud/fn/common/ftp/FtpSessionFactoryProperties.java
View File

@@ -48,6 +48,7 @@ public class FtpSessionFactoryProperties {
*/
private String username;
/**
* The password to use to connect to the server.
*/

12
common/spring-function-test-support/src/main/java/org/springframework/cloud/fn/test/support/file/remote/RemoteFileTestSupport.java
View File

@@ -80,13 +80,14 @@ public abstract class RemoteFileTestSupport {
* localTarget/
* </pre>
*
* The intent is tests retrieve from remoteSource and verify arrival in localTarget or send from localSource and verify
* arrival in remoteTarget.
* The intent is tests retrieve from remoteSource and verify arrival in localTarget or
* send from localSource and verify arrival in remoteTarget.
* <p>
* Subclasses can change 'remote' in these names by overriding {@link #prefix()} or override this method completely to
* create a different structure.
* Subclasses can change 'remote' in these names by overriding {@link #prefix()} or
* override this method completely to create a different structure.
* <p>
* While a single server exists for all tests, the directory structure is rebuilt for each test.
* While a single server exists for all tests, the directory structure is rebuilt for
* each test.
* @throws IOException IO Exception.
*/
@BeforeEach
@@ -151,4 +152,5 @@ public abstract class RemoteFileTestSupport {
protected String prefix() {
return "remote";
}
}

5
common/spring-function-test-support/src/main/java/org/springframework/cloud/fn/test/support/ftp/FtpTestSupport.java
View File

@@ -79,14 +79,12 @@ public class FtpTestSupport extends RemoteFileTestSupport {
private TestUserManager(String homeDirectory) {
this.testUser = new BaseUser();
this.testUser.setAuthorities(Arrays.asList(new ConcurrentLoginPermission(1024, 1024),
new WritePermission(),
this.testUser.setAuthorities(Arrays.asList(new ConcurrentLoginPermission(1024, 1024), new WritePermission(),
new TransferRatePermission(1024, 1024)));
this.testUser.setHomeDirectory(homeDirectory);
this.testUser.setName("TEST_USER");
}
@Override
public User getUserByName(String s) throws FtpException {
return this.testUser;
@@ -126,4 +124,5 @@ public class FtpTestSupport extends RemoteFileTestSupport {
}
}
}

13
common/spring-function-test-support/src/main/java/org/springframework/cloud/fn/test/support/sftp/SftpTestSupport.java
View File

@@ -58,8 +58,13 @@ public class SftpTestSupport extends RemoteFileTestSupport {
@BeforeAll
public static void createServer() throws Exception {
server = SshServer.setUpDefaultServer();
server.setPasswordAuthenticator((username, password, session) ->
StringUtils.hasText(password) && !"badPassword".equals(password)); // fail if pub key validation failed
server.setPasswordAuthenticator(
(username, password, session) -> StringUtils.hasText(password) && !"badPassword".equals(password)); // fail
// if
// pub
// key
// validation
// failed
server.setPublickeyAuthenticator((username, key, session) -> key.equals(decodePublicKey("id_rsa_pp.pub")));
server.setPort(0);
server.setKeyPairProvider(new SimpleGeneratorHostKeyProvider(new File("hostkey.ser").toPath()));
@@ -67,8 +72,7 @@ public class SftpTestSupport extends RemoteFileTestSupport {
server.setFileSystemFactory(new VirtualFileSystemFactory(remoteTemporaryFolder));
server.start();
System.setProperty("sftp.factory.port", String.valueOf(server.getPort()));
System.setProperty("sftp.consumer.localDir",
localTemporaryFolder + File.separator + "localTarget");
System.setProperty("sftp.consumer.localDir", localTemporaryFolder + File.separator + "localTarget");
}
@AfterAll
@@ -117,4 +121,5 @@ public class SftpTestSupport extends RemoteFileTestSupport {
bb.get(bytes);
return new BigInteger(bytes);
}
}

1
common/spring-function-test-support/src/main/java/org/springframework/cloud/fn/test/support/websocket/WebsocketConsumerClientHandler.java
View File

@@ -57,4 +57,5 @@ public class WebsocketConsumerClientHandler extends AbstractWebSocketHandler {
public List<String> getReceivedMessages() {
return receivedMessages;
}
}

3
common/spring-function-test-support/src/main/java/org/springframework/cloud/fn/test/support/xmpp/XmppTestContainerSupport.java
View File

@@ -57,8 +57,7 @@ public interface XmppTestContainerSupport {
/**
* The container.
*/
GenericContainer<?> XMPP_CONTAINER = new GenericContainer<>("fishbowler/openfire:v4.7.0")
.withExposedPorts(5222)
GenericContainer<?> XMPP_CONTAINER = new GenericContainer<>("fishbowler/openfire:v4.7.0").withExposedPorts(5222)
.withClasspathResourceMapping("xmpp/conf", "/var/lib/openfire/conf", BindMode.READ_ONLY)
.withCommand("-demoboot")
.withStartupTimeout(Duration.ofSeconds(120))

2
common/spring-metadata-store-common/src/main/java/org/springframework/cloud/fn/common/metadata/store/MetadataStoreAutoConfiguration.java
View File

@@ -47,7 +47,6 @@ import org.springframework.jdbc.core.JdbcTemplate;
* @author Artem Bilan
* @author David Turanski
* @author Corneil du Plessis
*
* @since 2.0.2
*/
@AutoConfiguration
@@ -170,6 +169,7 @@ public class MetadataStoreAutoConfiguration {
@ConditionalOnProperty(prefix = "metadata.store", name = "type", havingValue = "jdbc")
static class Jdbc {
@Bean
@ConditionalOnMissingBean
public ConcurrentMetadataStore jdbcMetadataStore(JdbcTemplate jdbcTemplate,

15
common/spring-metadata-store-common/src/main/java/org/springframework/cloud/fn/common/metadata/store/MetadataStoreProperties.java
View File

@@ -32,19 +32,16 @@ import org.springframework.integration.redis.metadata.RedisMetadataStore;
*/
@ConfigurationProperties("metadata.store")
public class MetadataStoreProperties {
enum StoreType {
mongodb,
redis,
dynamodb,
jdbc,
zookeeper,
hazelcast,
memory
mongodb, redis, dynamodb, jdbc, zookeeper, hazelcast, memory
}
/**
* Indicates the type of metadata store to configure (default is 'memory').
* You must include the corresponding Spring Integration dependency to use a persistent store.
* Indicates the type of metadata store to configure (default is 'memory'). You must
* include the corresponding Spring Integration dependency to use a persistent store.
*/
private StoreType type = StoreType.memory;

53
common/spring-metadata-store-common/src/test/java/org/springframework/cloud/fn/common/metadata/store/MetadataStoreAutoConfigurationTests.java
View File

@@ -56,41 +56,30 @@ import static org.mockito.Mockito.mock;
/**
* @author Artem Bilan
* @author Corneil du Plessis
*
* @since 2.0.2
*/
public class MetadataStoreAutoConfigurationTests {
private final static List<Class<? extends ConcurrentMetadataStore>> METADATA_STORE_CLASSES =
List.of(
RedisMetadataStore.class,
MongoDbMetadataStore.class,
JdbcMetadataStore.class,
ZookeeperMetadataStore.class,
HazelcastMetadataStore.class,
DynamoDbMetadataStore.class,
SimpleMetadataStore.class
);
private final static List<Class<? extends ConcurrentMetadataStore>> METADATA_STORE_CLASSES = List.of(
RedisMetadataStore.class, MongoDbMetadataStore.class, JdbcMetadataStore.class, ZookeeperMetadataStore.class,
HazelcastMetadataStore.class, DynamoDbMetadataStore.class, SimpleMetadataStore.class);
@ParameterizedTest
@MethodSource
public void testMetadataStore(Class<? extends ConcurrentMetadataStore> classToInclude) {
ApplicationContextRunner contextRunner =
new ApplicationContextRunner()
.withUserConfiguration(TestConfiguration.class)
.withPropertyValues("metadata.store.type=" +
classToInclude.getSimpleName()
.replaceFirst("MetadataStore", "")
.toLowerCase()
.replaceFirst("simple", "memory"))
.withClassLoader(filteredClassLoaderBut(classToInclude));
contextRunner
.run(context -> {
assertThat(context.getBeansOfType(MetadataStore.class)).hasSize(1);
ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withUserConfiguration(TestConfiguration.class)
.withPropertyValues("metadata.store.type=" + classToInclude.getSimpleName()
.replaceFirst("MetadataStore", "")
.toLowerCase()
.replaceFirst("simple", "memory"))
.withClassLoader(filteredClassLoaderBut(classToInclude));
contextRunner.run(context -> {
assertThat(context.getBeansOfType(MetadataStore.class)).hasSize(1);
assertThat(context.getBeanNamesForType(classToInclude))
.containsOnlyOnce(Introspector.decapitalize(classToInclude.getSimpleName()));
});
assertThat(context.getBeanNamesForType(classToInclude))
.containsOnlyOnce(Introspector.decapitalize(classToInclude.getSimpleName()));
});
}
static List<Class<? extends ConcurrentMetadataStore>> testMetadataStore() {
@@ -98,10 +87,9 @@ public class MetadataStoreAutoConfigurationTests {
}
private static FilteredClassLoader filteredClassLoaderBut(Class<? extends ConcurrentMetadataStore> classToInclude) {
return new FilteredClassLoader(
METADATA_STORE_CLASSES.stream()
.filter(Predicate.isEqual(classToInclude).negate())
.toArray(Class<?>[]::new));
return new FilteredClassLoader(METADATA_STORE_CLASSES.stream()
.filter(Predicate.isEqual(classToInclude).negate())
.toArray(Class<?>[]::new));
}
@Configuration
@@ -126,9 +114,8 @@ public class MetadataStoreAutoConfigurationTests {
@Bean
public static DynamoDbAsyncClient dynamoDB() {
DynamoDbAsyncClient dynamoDb = mock(DynamoDbAsyncClient.class);
willReturn(CompletableFuture.completedFuture(DescribeTableResponse.builder().build()))
.given(dynamoDb)
.describeTable(ArgumentMatchers.<Consumer<DescribeTableRequest.Builder>>any());
willReturn(CompletableFuture.completedFuture(DescribeTableResponse.builder().build())).given(dynamoDb)
.describeTable(ArgumentMatchers.<Consumer<DescribeTableRequest.Builder>>any());
return dynamoDb;
}

3
common/spring-tcp-common/src/main/java/org/springframework/cloud/fn/common/tcp/EncoderDecoderFactoryBean.java
View File

@@ -29,8 +29,7 @@ import org.springframework.integration.ip.tcp.serializer.ByteArrayStxEtxSerializ
import org.springframework.util.Assert;
/**
* Factory bean for an encoder/decoder based on
* {@link Encoding}.
* Factory bean for an encoder/decoder based on {@link Encoding}.
*
* @author Gary Russell
* @author Christian Tzolov

2
common/spring-tcp-common/src/main/java/org/springframework/cloud/fn/common/tcp/Encoding.java
View File

@@ -21,6 +21,7 @@ package org.springframework.cloud.fn.common.tcp;
* @author Christian Tzolov
*/
public enum Encoding {
/**
* CRLF encoding.
*/
@@ -53,4 +54,5 @@ public enum Encoding {
* L4 encoding.
*/
L4;
}

5
common/spring-tcp-common/src/main/java/org/springframework/cloud/fn/common/tcp/TcpConnectionFactoryProperties.java
View File

@@ -34,8 +34,8 @@ public class TcpConnectionFactoryProperties {
private int port = 1234;
/**
* Perform a reverse DNS lookup on the remote IP Address; if false,
* just the IP address is included in the message headers.
* Perform a reverse DNS lookup on the remote IP Address; if false, just the IP
* address is included in the message headers.
*/
private boolean reverseLookup = false;
@@ -93,4 +93,5 @@ public class TcpConnectionFactoryProperties {
public void setReverseLookup(boolean reverseLookup) {
this.reverseLookup = reverseLookup;
}
}

22
common/spring-tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/AbstractGraphRunner.java
View File

@@ -34,8 +34,9 @@ public abstract class AbstractGraphRunner implements Function<Map<String, Tensor
public abstract Session doGetSession();
/**
* Names expected in the named Tensor inside the input {@link AbstractGraphRunner#apply(Map)}.
* If the apply method will fail if the input map is missing some of the feedNames.
* Names expected in the named Tensor inside the input
* {@link AbstractGraphRunner#apply(Map)}. If the apply method will fail if the input
* map is missing some of the feedNames.
*/
private final List<String> feedNames;
@@ -45,8 +46,9 @@ public abstract class AbstractGraphRunner implements Function<Map<String, Tensor
private final List<String> fetchNames;
/**
* When set and the input takes a single feed, then the name of the input tensor is automatically mapped
* to the expected input name. E.g. no need to rename the input names explicitly.
* When set and the input takes a single feed, then the name of the input tensor is
* automatically mapped to the expected input name. E.g. no need to rename the input
* names explicitly.
*/
private boolean autoBinding;
@@ -69,8 +71,8 @@ public abstract class AbstractGraphRunner implements Function<Map<String, Tensor
}
if (this.isAutoBinding() && (feeds.size() != 1)) {
throw new IllegalArgumentException("Feed auto-binding expects a " +
"single feed tensors but found: " + feeds);
throw new IllegalArgumentException(
"Feed auto-binding expects a " + "single feed tensors but found: " + feeds);
}
Session.Runner runner = this.doGetSession().runner();
@@ -127,8 +129,8 @@ public abstract class AbstractGraphRunner implements Function<Map<String, Tensor
public AbstractGraphRunner enableAutoBinding() {
if (this.getFeedNames().size() != 1) {
throw new IllegalArgumentException("Auto-binding is permitted for Graphs with single input feed, but " +
" found: " + this.getFeedNames());
throw new IllegalArgumentException("Auto-binding is permitted for Graphs with single input feed, but "
+ " found: " + this.getFeedNames());
}
this.autoBinding = true;
return this;
@@ -136,7 +138,7 @@ public abstract class AbstractGraphRunner implements Function<Map<String, Tensor
@Override
public String toString() {
return String.format("(%s) -> (%s)", String.join(",", this.feedNames),
String.join(",", this.fetchNames));
return String.format("(%s) -> (%s)", String.join(",", this.feedNames), String.join(",", this.fetchNames));
}
}

1
common/spring-tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/AutoCloseableSession.java
View File

@@ -60,4 +60,5 @@ class AutoCloseableSession implements AutoCloseable {
protected void doClose() {
}
}

41
common/spring-tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/Functions.java
View File

@@ -33,26 +33,28 @@ public final class Functions {
}
/**
* On every function call enrich the input tensorMap with an addition (tensorName, tensor) pair.
*
* On every function call enrich the input tensorMap with an addition (tensorName,
* tensor) pair.
* @param tensorName tensor key to use in the map
* @param tensor new Tensor to add to the map
* @return Returns a copy of the input tensorMap enriched with the provided (tensorName, tensor).
* @return Returns a copy of the input tensorMap enriched with the provided
* (tensorName, tensor).
*/
public static Function<Map<String, Tensor<?>>, Map<String, Tensor<?>>> enrichWith(
String tensorName, Tensor<?> tensor) {
public static Function<Map<String, Tensor<?>>, Map<String, Tensor<?>>> enrichWith(String tensorName,
Tensor<?> tensor) {
return tensorMap -> enrich(tensorMap, tensorName, tensor);
}
/**
* On function call retrieves a named tensor from the provided {@link GraphRunnerMemory} and uses it to enrich
* the input tensorMap.
* On function call retrieves a named tensor from the provided
* {@link GraphRunnerMemory} and uses it to enrich the input tensorMap.
* @param memory GraphRunnerMemory to retrieve the tensor from
* @param tensorName name of the tensor in GraphRunnerMemory to retrieve.
* @return Returns copy of the input tensorMap enriched with the tensor from the memory.
* @return Returns copy of the input tensorMap enriched with the tensor from the
* memory.
*/
public static Function<Map<String, Tensor<?>>, Map<String, Tensor<?>>> enrichFromMemory(
GraphRunnerMemory memory, String tensorName) {
public static Function<Map<String, Tensor<?>>, Map<String, Tensor<?>>> enrichFromMemory(GraphRunnerMemory memory,
String tensorName) {
return tensorMap -> enrich(tensorMap, tensorName, memory.getTensorMap().get(tensorName));
}
@@ -64,10 +66,10 @@ public final class Functions {
/**
* Renames the tensor names in the incoming tensorMap with the providing mappings.
*
* @param mapping Pairs of From and To names. E.g. fromName1, toName1, fromName2, toName2, ... fromNameN, toNameN
* Must be an even number.
* @return Map that renames the input tensorMap entries according to the mapping provided
* @param mapping Pairs of From and To names. E.g. fromName1, toName1, fromName2,
* toName2, ... fromNameN, toNameN Must be an even number.
* @return Map that renames the input tensorMap entries according to the mapping
* provided
*/
public static Function<Map<String, Tensor<?>>, Map<String, Tensor<?>>> rename(String... mapping) {
@@ -76,11 +78,10 @@ public final class Functions {
mappingMap.put(mapping[i], mapping[i + 1]);
}
return tensorMap -> tensorMap.entrySet().stream()
.filter(e -> mappingMap.containsKey(e.getKey()))
.collect(Collectors.toMap(
kv -> mappingMap.get(kv.getKey()),
kv -> kv.getValue()
));
return tensorMap -> tensorMap.entrySet()
.stream()
.filter(e -> mappingMap.containsKey(e.getKey()))
.collect(Collectors.toMap(kv -> mappingMap.get(kv.getKey()), kv -> kv.getValue()));
}
}

2
common/spring-tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphDefinition.java
View File

@@ -23,5 +23,7 @@ import org.tensorflow.op.Ops;
*/
@FunctionalInterface
public interface GraphDefinition {
void defineGraph(Ops tf);
}

12
common/spring-tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphRunner.java
View File

@@ -24,13 +24,13 @@ import org.tensorflow.SavedModelBundle;
import org.tensorflow.Session;
import org.tensorflow.op.Ops;
/**
* @author Christian Tzolov
*/
public class GraphRunner extends AbstractGraphRunner implements AutoCloseable {
private SavedModelBundle savedModelBundle;
private AutoCloseableSession autoCloseableSession;
public GraphRunner(List<String> feedNames, String fetchedName) {
@@ -49,7 +49,6 @@ public class GraphRunner extends AbstractGraphRunner implements AutoCloseable {
super(feedNames, fetchedNames);
}
@Override
public Session doGetSession() {
@@ -69,8 +68,8 @@ public class GraphRunner extends AbstractGraphRunner implements AutoCloseable {
}
public GraphRunner withGraphDefinition(GraphDefinition graphDefinition) {
Validate.isTrue(this.savedModelBundle == null, "Either SavedModel or GraphDefinition can be set! " +
"SavedModelBundle is found: " + this.savedModelBundle);
Validate.isTrue(this.savedModelBundle == null, "Either SavedModel or GraphDefinition can be set! "
+ "SavedModelBundle is found: " + this.savedModelBundle);
this.autoCloseableSession = new AutoCloseableSession() {
@Override
@@ -83,8 +82,8 @@ public class GraphRunner extends AbstractGraphRunner implements AutoCloseable {
}
public GraphRunner withSavedModel(String savedModelDir, String... tags) {
Validate.isTrue(this.autoCloseableSession == null, "Either SavedModel or GraphDefinition can be set! " +
"AutoCloseableSession is found: " + this.autoCloseableSession);
Validate.isTrue(this.autoCloseableSession == null, "Either SavedModel or GraphDefinition can be set! "
+ "AutoCloseableSession is found: " + this.autoCloseableSession);
this.savedModelBundle = SavedModelBundle.load(savedModelDir, tags);
return this;
}
@@ -104,4 +103,5 @@ public class GraphRunner extends AbstractGraphRunner implements AutoCloseable {
this.autoCloseableSession.close();
}
}
}

5
common/spring-tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphRunnerMemory.java
View File

@@ -26,7 +26,6 @@ import org.tensorflow.Tensor;
import org.springframework.cloud.fn.common.tensorflow.util.AutoCloseables;
/**
* Keeps all tensorMap input parameters.
*/
@@ -47,7 +46,7 @@ public class GraphRunnerMemory implements Function<Map<String, Tensor<?>>, Map<S
@Override
public void close() {
AutoCloseables.all(this.tensorMap.get());
//this.tensorMap.get().clear();
// this.tensorMap.get().clear();
}
}
}

13
common/spring-tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/ProtoBufGraphDefinition.java
View File

@@ -58,12 +58,12 @@ public class ProtoBufGraphDefinition implements GraphDefinition {
: new ModelExtractor().getModel(this.modelLocation);
// Import the pre-trained model
((Graph) tf.scope().env()).importGraphDef(model);
//try {
// ((Graph) tf.scope().env()).importGraphDef(GraphDef.parseFrom(model));
//}
//catch (InvalidProtocolBufferException e) {
// throw new RuntimeException(e);
//}
// try {
// ((Graph) tf.scope().env()).importGraphDef(GraphDef.parseFrom(model));
// }
// catch (InvalidProtocolBufferException e) {
// throw new RuntimeException(e);
// }
Graph graph = ((Graph) tf.scope().env());
Iterator<Operation> ops = graph.operations();
@@ -71,4 +71,5 @@ public class ProtoBufGraphDefinition implements GraphDefinition {
System.out.println(ops.next().name());
}
}
}

251
common/spring-tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/deprecated/GraphicsUtils.java
View File

@@ -38,10 +38,10 @@ import org.apache.commons.io.IOUtils;
import org.springframework.core.io.DefaultResourceLoader;
import org.springframework.core.io.Resource;
/**
* Utility class used to provide some handy image manipulation functions. Among others it can provide contrast colors
* for image annotation labels and bounding boxes as well as functionality to draw later.
* Utility class used to provide some handy image manipulation functions. Among others it
* can provide contrast colors for image annotation labels and bounding boxes as well as
* functionality to draw later.
*
* @author Christian Tzolov
*/
@@ -70,295 +70,434 @@ public final class GraphicsUtils {
*/
public static final int TITLE_OFFSET = 3;
/**
* Predefined contrasting colors used when drawing multiple objects in the same image.
*/
public static final Color aliceblue = new Color(240, 248, 255); /* color */
/** Color. **/
public static final Color antiquewhite = new Color(250, 235, 215);
/** Color. **/
public static final Color aqua = new Color(0, 255, 255); // color
/** Color. **/
public static final Color aquamarine = new Color(127, 255, 212); // color
/** Color. **/
public static final Color azure = new Color(240, 255, 255); // color
/** Color. **/
public static final Color beige = new Color(245, 245, 220); // color
/** Color. **/
public static final Color bisque = new Color(255, 228, 196);
/** Color. **/
public static final Color black = new Color(0, 0, 0);
/** Color. **/
public static final Color blanchedalmond = new Color(255, 255, 205);
/** Color. **/
public static final Color blue = new Color(0, 0, 255);
/** Color. **/
public static final Color blueviolet = new Color(138, 43, 226);
/** Color. **/
public static final Color brown = new Color(165, 42, 42);
/** Color. **/
public static final Color burlywood = new Color(222, 184, 135);
/** Color. **/
public static final Color cadetblue = new Color(95, 158, 160);
/** Color. **/
public static final Color chartreuse = new Color(127, 255, 0);
/** Color. **/
public static final Color chocolate = new Color(210, 105, 30);
/** Color. **/
public static final Color coral = new Color(255, 127, 80);
/** Color. **/
public static final Color cornflowerblue = new Color(100, 149, 237);
/** Color. **/
public static final Color cornsilk = new Color(255, 248, 220);
/** Color. **/
public static final Color crimson = new Color(220, 20, 60);
/** Color. **/
public static final Color cyan = new Color(0, 255, 255);
/** Color. **/
public static final Color darkblue = new Color(0, 0, 139);
/** Color. **/
public static final Color darkcyan = new Color(0, 139, 139);
/** Color. **/
public static final Color darkgoldenrod = new Color(184, 134, 11);
/** Color. **/
public static final Color darkgray = new Color(169, 169, 169);
/** Color. **/
public static final Color darkgreen = new Color(0, 100, 0);
/** Color. **/
public static final Color darkkhaki = new Color(189, 183, 107);
/** Color. **/
public static final Color darkmagenta = new Color(139, 0, 139);
/** Color. **/
public static final Color darkolivegreen = new Color(85, 107, 47);
/** Color. **/
public static final Color darkorange = new Color(255, 140, 0);
/** Color. **/
public static final Color darkorchid = new Color(153, 50, 204);
/** Color. **/
public static final Color darkred = new Color(139, 0, 0);
/** Color. **/
public static final Color darksalmon = new Color(233, 150, 122);
/** Color. **/
public static final Color darkseagreen = new Color(143, 188, 143);
/** Color. **/
public static final Color darkslateblue = new Color(72, 61, 139);
/** Color. **/
public static final Color darkslategray = new Color(47, 79, 79);
/** Color. **/
public static final Color darkturquoise = new Color(0, 206, 209);
/** Color. **/
public static final Color darkviolet = new Color(148, 0, 211);
/** Color. **/
public static final Color deeppink = new Color(255, 20, 147);
/** Color. **/
public static final Color deepskyblue = new Color(0, 191, 255);
/** Color. **/
public static final Color dimgray = new Color(105, 105, 105);
/** Color. **/
public static final Color dodgerblue = new Color(30, 144, 255);
/** Color. **/
public static final Color firebrick = new Color(178, 34, 34);
/** Color. **/
public static final Color floralwhite = new Color(255, 250, 240);
/** Color. **/
public static final Color forestgreen = new Color(34, 139, 34);
/** Color. **/
public static final Color fuchsia = new Color(255, 0, 255);
/** Color. **/
public static final Color gainsboro = new Color(220, 220, 220);
/** Color. **/
public static final Color ghostwhite = new Color(248, 248, 255);
/** Color. **/
public static final Color gold = new Color(255, 215, 0);
/** Color. **/
public static final Color goldenrod = new Color(218, 165, 32);
/** Color. **/
public static final Color gray = new Color(128, 128, 128);
/** Color. **/
public static final Color green = new Color(0, 128, 0);
/** Color. **/
public static final Color greenyellow = new Color(173, 255, 47);
/** Color. **/
public static final Color honeydew = new Color(240, 255, 240);
/** Color. **/
public static final Color hotpink = new Color(255, 105, 180);
/** Color. **/
public static final Color indianred = new Color(205, 92, 92);
/** Color. **/
public static final Color indigo = new Color(75, 0, 130);
/** Color. **/
public static final Color ivory = new Color(255, 240, 240);
/** Color. **/
public static final Color khaki = new Color(240, 230, 140);
/** Color. **/
public static final Color lavender = new Color(230, 230, 250);
/** Color. **/
public static final Color lavenderblush = new Color(255, 240, 245);
/** Color. **/
public static final Color lawngreen = new Color(124, 252, 0);
/** Color. **/
public static final Color lemonchiffon = new Color(255, 250, 205);
/** Color. **/
public static final Color lightblue = new Color(173, 216, 230);
/** Color. **/
public static final Color lightcoral = new Color(240, 128, 128);
/** Color. **/
public static final Color lightcyan = new Color(224, 255, 255);
/** Color. **/
public static final Color lightgoldenrodyellow = new Color(250, 250, 210);
/** Color. **/
public static final Color lightgreen = new Color(144, 238, 144);
/** Color. **/
public static final Color lightgrey = new Color(211, 211, 211);
/** Color. **/
public static final Color lightpink = new Color(255, 182, 193);
/** Color. **/
public static final Color lightsalmon = new Color(255, 160, 122);
/** Color. **/
public static final Color lightseagreen = new Color(32, 178, 170);
/** Color. **/
public static final Color lightskyblue = new Color(135, 206, 250);
/** Color. **/
public static final Color lightslategray = new Color(119, 136, 153);
/** Color. **/
public static final Color lightsteelblue = new Color(176, 196, 222);
/** Color. **/
public static final Color lightyellow = new Color(255, 255, 224);
/** Color. **/
public static final Color lime = new Color(0, 255, 0);
/** Color. **/
public static final Color limegreen = new Color(50, 205, 50);
/** Color. **/
public static final Color linen = new Color(250, 240, 230);
/** Color. **/
public static final Color magenta = new Color(255, 0, 255);
/** Color. **/
public static final Color maroon = new Color(128, 0, 0);
/** Color. **/
public static final Color mediumaquamarine = new Color(102, 205, 170);
/** Color. **/
public static final Color mediumblue = new Color(0, 0, 205);
/** Color. **/
public static final Color mediumorchid = new Color(186, 85, 211);
/** Color. **/
public static final Color mediumpurple = new Color(147, 112, 219);
/** Color. **/
public static final Color mediumseagreen = new Color(60, 179, 113);
/** Color. **/
public static final Color mediumslateblue = new Color(123, 104, 238);
/** Color. **/
public static final Color mediumspringgreen = new Color(0, 250, 154);
/** Color. **/
public static final Color mediumturquoise = new Color(72, 209, 204);
/** Color. **/
public static final Color mediumvioletred = new Color(199, 21, 133);
/** Color. **/
public static final Color midnightblue = new Color(25, 25, 112);
/** Color. **/
public static final Color mintcream = new Color(245, 255, 250);
/** Color. **/
public static final Color mistyrose = new Color(255, 228, 225);
/** Color. **/
public static final Color mocassin = new Color(255, 228, 181);
/** Color. **/
public static final Color navajowhite = new Color(255, 222, 173);
/** Color. **/
public static final Color navy = new Color(0, 0, 128);
/** Color. **/
public static final Color oldlace = new Color(253, 245, 230);
/** Color. **/
public static final Color olive = new Color(128, 128, 0);
/** Color. **/
public static final Color olivedrab = new Color(107, 142, 35);
/** Color. **/
public static final Color orange = new Color(255, 165, 0);
/** Color. **/
public static final Color orangered = new Color(255, 69, 0);
/** Color. **/
public static final Color orchid = new Color(218, 112, 214);
/** Color. **/
public static final Color palegoldenrod = new Color(238, 232, 170);
/** Color. **/
public static final Color palegreen = new Color(152, 251, 152);
/** Color. **/
public static final Color paleturquoise = new Color(175, 238, 238);
/** Color. **/
public static final Color palevioletred = new Color(219, 112, 147);
/** Color. **/
public static final Color papayawhip = new Color(255, 239, 213);
/** Color. **/
public static final Color peachpuff = new Color(255, 218, 185);
/** Color. **/
public static final Color peru = new Color(205, 133, 63);
/** Color. **/
public static final Color pink = new Color(255, 192, 203);
/** Color. **/
public static final Color plum = new Color(221, 160, 221);
/** Color. **/
public static final Color powderblue = new Color(176, 224, 230);
/** Color. **/
public static final Color purple = new Color(128, 0, 128);
/** Color. **/
public static final Color red = new Color(255, 0, 0);
/** Color. **/
public static final Color rosybrown = new Color(188, 143, 143);
/** Color. **/
public static final Color royalblue = new Color(65, 105, 225);
/** Color. **/
public static final Color saddlebrown = new Color(139, 69, 19);
/** Color. **/
public static final Color salmon = new Color(250, 128, 114);
/** Color. **/
public static final Color sandybrown = new Color(244, 164, 96);
/** Color. **/
public static final Color seagreen = new Color(46, 139, 87);
/** Color. **/
public static final Color seashell = new Color(255, 245, 238);
/** Color. **/
public static final Color sienna = new Color(160, 82, 45);
/** Color. **/
public static final Color silver = new Color(192, 192, 192);
/** Color. **/
public static final Color skyblue = new Color(135, 206, 235);
/** Color. **/
public static final Color slateblue = new Color(106, 90, 205);
/** Color. **/
public static final Color slategray = new Color(112, 128, 144);
/** Color. **/
public static final Color snow = new Color(255, 250, 250);
/** Color. **/
public static final Color springgreen = new Color(0, 255, 127);
/** Color. **/
public static final Color steelblue = new Color(70, 138, 180);
/** Color. **/
public static final Color tan = new Color(210, 180, 140);
/** Color. **/
public static final Color teal = new Color(0, 128, 128);
/** Color. **/
public static final Color thistle = new Color(216, 191, 216);
/** Color. **/
public static final Color tomato = new Color(253, 99, 71);
/** Color. **/
public static final Color turquoise = new Color(64, 224, 208);
/** Color. **/
public static final Color violet = new Color(238, 130, 238);
/** Color. **/
public static final Color wheat = new Color(245, 222, 179);
/** Color. **/
public static final Color white = new Color(255, 255, 255);
/** Color. **/
public static final Color whitesmoke = new Color(245, 245, 245);
/** Color. **/
public static final Color yellow = new Color(255, 255, 0);
/** Color. **/
public static final Color yellowgreen = new Color(154, 205, 50);
/**
* Limbs color list.
*/
public static final Color[] LIMBS_COLORS = new Color[] {
new Color(153, 0, 0), // 0 (1 -> 2)
public static final Color[] LIMBS_COLORS = new Color[] { new Color(153, 0, 0), // 0 (1
// ->
// 2)
new Color(153, 51, 0), // 1 (1 -> 5)
new Color(153, 102, 0), // 2 (2 -> 3)
new Color(153, 153, 0), // 3 (3 -> 4)
@@ -383,42 +522,31 @@ public final class GraphicsUtils {
/**
* Constants lists.
*/
private static final Color[] CLASS_COLOR = new Color[] {
aliceblue, chartreuse, aqua, aquamarine, azure, beige, bisque,
blanchedalmond, blueviolet, burlywood, cadetblue, antiquewhite,
chocolate, coral, cornflowerblue, cornsilk, crimson, cyan,
darkcyan, darkgoldenrod, darkgray, darkkhaki, darkorange,
darkorchid, darksalmon, darkseagreen, darkturquoise, darkviolet,
deeppink, deepskyblue, dodgerblue, firebrick, floralwhite,
forestgreen, fuchsia, gainsboro, ghostwhite, gold, goldenrod,
salmon, tan, honeydew, hotpink, indianred, ivory, khaki,
lavender, lavenderblush, lawngreen, lemonchiffon, lightblue,
lightcoral, lightcyan, lightgoldenrodyellow, lightgreen, lightgrey,
lightgreen, lightpink, lightsalmon, lightseagreen, lightskyblue,
lightslategray, lightslategray, lightsteelblue, lightyellow, lime,
limegreen, linen, magenta, mediumaquamarine, mediumorchid,
mediumpurple, mediumseagreen, mediumslateblue, mediumspringgreen,
mediumturquoise, mediumvioletred, mintcream, mistyrose, mocassin,
navajowhite, oldlace, olive, olivedrab, orange, orangered,
orchid, palegoldenrod, palegreen, paleturquoise, palevioletred,
papayawhip, peachpuff, peru, pink, plum, powderblue, purple,
red, rosybrown, royalblue, saddlebrown, green, sandybrown,
seagreen, seashell, sienna, silver, skyblue, slateblue,
slategray, slategray, snow, springgreen, steelblue, greenyellow,
teal, thistle, tomato, turquoise, violet, wheat, white,
whitesmoke, yellow, yellowgreen
};
private static final Color[] CLASS_COLOR = new Color[] { aliceblue, chartreuse, aqua, aquamarine, azure, beige,
bisque, blanchedalmond, blueviolet, burlywood, cadetblue, antiquewhite, chocolate, coral, cornflowerblue,
cornsilk, crimson, cyan, darkcyan, darkgoldenrod, darkgray, darkkhaki, darkorange, darkorchid, darksalmon,
darkseagreen, darkturquoise, darkviolet, deeppink, deepskyblue, dodgerblue, firebrick, floralwhite,
forestgreen, fuchsia, gainsboro, ghostwhite, gold, goldenrod, salmon, tan, honeydew, hotpink, indianred,
ivory, khaki, lavender, lavenderblush, lawngreen, lemonchiffon, lightblue, lightcoral, lightcyan,
lightgoldenrodyellow, lightgreen, lightgrey, lightgreen, lightpink, lightsalmon, lightseagreen,
lightskyblue, lightslategray, lightslategray, lightsteelblue, lightyellow, lime, limegreen, linen, magenta,
mediumaquamarine, mediumorchid, mediumpurple, mediumseagreen, mediumslateblue, mediumspringgreen,
mediumturquoise, mediumvioletred, mintcream, mistyrose, mocassin, navajowhite, oldlace, olive, olivedrab,
orange, orangered, orchid, palegoldenrod, palegreen, paleturquoise, palevioletred, papayawhip, peachpuff,
peru, pink, plum, powderblue, purple, red, rosybrown, royalblue, saddlebrown, green, sandybrown, seagreen,
seashell, sienna, silver, skyblue, slateblue, slategray, slategray, snow, springgreen, steelblue,
greenyellow, teal, thistle, tomato, turquoise, violet, wheat, white, whitesmoke, yellow, yellowgreen };
/**
* List of constants.
*/
public static final Color[] CLASS_COLOR2 = new Color[] {
yellow, yellowgreen, turquoise, springgreen, skyblue, slateblue, red, violet, olivedrab, royalblue,
darkorange, mediumblue, deeppink, chartreuse, orchid, palegreen, aqua, orange, navy
};
public static final Color[] CLASS_COLOR2 = new Color[] { yellow, yellowgreen, turquoise, springgreen, skyblue,
slateblue, red, violet, olivedrab, royalblue, darkorange, mediumblue, deeppink, chartreuse, orchid,
palegreen, aqua, orange, navy };
/**
* Return different color for each Id. It rotates when the ID exceeds the number of predefined colors.
* Return different color for each Id. It rotates when the ID exceeds the number of
* predefined colors.
* @param id the unique id to pick color for.
* @return a distinct color computed from the input #id
*/
@@ -427,17 +555,18 @@ public final class GraphicsUtils {
}
/**
* Augments the input image fromMemory a labeled rectangle (e.g. bounding box) fromMemory coordinates: (x1, y1, x2, y2).
*
* Augments the input image fromMemory a labeled rectangle (e.g. bounding box)
* fromMemory coordinates: (x1, y1, x2, y2).
* @param image Input image to be augmented fromMemory labeled rectangle.
* @param cid Unique id used to select the color of the rectangle. Used only if the colorAgnostic is set to false.
* @param cid Unique id used to select the color of the rectangle. Used only if the
* colorAgnostic is set to false.
* @param title rectangle title
* @param x1 top left corner for the bounding box
* @param y1 top left corner for the bounding box
* @param x2 bottom right corner for the bounding box
* @param y2 bottom right corner for the bounding box
* @param colorAgnostic If set to false the cid is used to select the bounding box color. Uses the
* AGNOSTIC_COLOR otherwise.
* @param colorAgnostic If set to false the cid is used to select the bounding box
* color. Uses the AGNOSTIC_COLOR otherwise.
*/
public static void drawBoundingBox(BufferedImage image, int cid, String title, int x1, int y1, int x2, int y2,
boolean colorAgnostic) {
@@ -459,8 +588,7 @@ public final class GraphicsUtils {
Rectangle2D rect = fontMetrics.getStringBounds(title, g);
g.setColor(labelColor);
g.fillRect(x1, y1 - fontMetrics.getAscent(),
(int) rect.getWidth() + 2 * TITLE_OFFSET, (int) rect.getHeight());
g.fillRect(x1, y1 - fontMetrics.getAscent(), (int) rect.getWidth() + 2 * TITLE_OFFSET, (int) rect.getHeight());
g.setColor(getTextColor(labelColor));
g.drawString(title, x1 + TITLE_OFFSET, y1);
@@ -472,13 +600,13 @@ public final class GraphicsUtils {
* @return a text color, that contrast to the given background color.
*/
private static Color getTextColor(Color backGroundColor) {
double y = (299 * backGroundColor.getRed() + 587 * backGroundColor.getGreen() +
114 * backGroundColor.getBlue()) / 1000;
double y = (299 * backGroundColor.getRed() + 587 * backGroundColor.getGreen() + 114 * backGroundColor.getBlue())
/ 1000;
return y >= 128 ? Color.black : Color.white;
}
public static BufferedImage createMaskImage(float[][] maskPixels,
int scaledWidth, int scaledHeight, Color maskColor) {
public static BufferedImage createMaskImage(float[][] maskPixels, int scaledWidth, int scaledHeight,
Color maskColor) {
int maskWidth = maskPixels.length;
int maskHeight = maskPixels[0].length;
@@ -500,7 +628,6 @@ public final class GraphicsUtils {
/**
* Converts an gray scale (e.g. value between 0 to 1) into ARGB.
*
* @param grayScale - value between 0 and 1
* @param maskColor - desired mask color
* @return Returns a ARGB color based on the grayscale and the mask colors
@@ -518,14 +645,14 @@ public final class GraphicsUtils {
}
private static float col(int channelColor, float grayScale) {
//return ((float) channelColor / 255) * grayScale;
// return ((float) channelColor / 255) * grayScale;
return ((float) channelColor / 255);
}
public static BufferedImage toBufferedImage(Image img) {
//if (img instanceof BufferedImage) {
// return (BufferedImage) img;
//}
// if (img instanceof BufferedImage) {
// return (BufferedImage) img;
// }
// Create a buffered image fromMemory transparency
BufferedImage bimage = new BufferedImage(img.getWidth(null), img.getHeight(null), BufferedImage.TYPE_INT_ARGB);
@@ -540,22 +667,20 @@ public final class GraphicsUtils {
}
public static BufferedImage overlayImages(BufferedImage bgImage, BufferedImage fgImage, int fgX, int fgY) {
// Foreground image width and height cannot be greater than background image width and height.
if (fgImage.getHeight() > bgImage.getHeight()
|| fgImage.getWidth() > fgImage.getWidth()) {
throw new IllegalArgumentException(
"Foreground Image Is Bigger In One or Both Dimensions"
+ "nCannot proceed fromMemory overlay."
+ "nn Please use smaller Image for foreground");
// Foreground image width and height cannot be greater than background image width
// and height.
if (fgImage.getHeight() > bgImage.getHeight() || fgImage.getWidth() > fgImage.getWidth()) {
throw new IllegalArgumentException("Foreground Image Is Bigger In One or Both Dimensions"
+ "nCannot proceed fromMemory overlay." + "nn Please use smaller Image for foreground");
}
// Create a Graphics from the background image
// Create a Graphics from the background image
Graphics2D g = bgImage.createGraphics();
//Set Antialias Rendering
// Set Antialias Rendering
g.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
//Draw background image at location (0,0)
// Draw background image at location (0,0)
g.drawImage(bgImage, 0, 0, null);
// Draw foreground image at location (fgX,fgy)
@@ -567,7 +692,6 @@ public final class GraphicsUtils {
/**
* Convert {@link BufferedImage} to byte array.
*
* @param image the image to be converted
* @param format the output image format
* @return New array of bytes
@@ -594,7 +718,6 @@ public final class GraphicsUtils {
}
/**
*
* @param bufferedImage buffer to be converted in to raw array
* @return flat byte array representing the buffered image
*/

1
common/spring-tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/deprecated/JsonMapperFunction.java
View File

@@ -43,4 +43,5 @@ public class JsonMapperFunction implements Function<Object, String> {
return "ERROR";
}
}

17
common/spring-tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/deprecated/TensorFlowService.java
View File

@@ -14,7 +14,6 @@
* limitations under the License.
*/
package org.springframework.cloud.fn.common.tensorflow.deprecated;
import java.util.HashMap;
@@ -42,7 +41,9 @@ public class TensorFlowService implements Function<Map<String, Tensor<?>>, Map<S
private static final Log logger = LogFactory.getLog(TensorFlowService.class);
private final Session session;
private final List<String> fetchedNames;
private final boolean autoCloseFeedTensors;
public TensorFlowService(Resource modelLocation, List<String> fetchedNames) {
@@ -63,18 +64,19 @@ public class TensorFlowService implements Function<Map<String, Tensor<?>>, Map<S
this.autoCloseFeedTensors = autoCloseFeedTensors;
this.fetchedNames = fetchedNames;
Graph graph = new Graph();
byte[] model = cacheModel ? new CachedModelExtractor().getModel(modelLocation) : new ModelExtractor().getModel(modelLocation);
byte[] model = cacheModel ? new CachedModelExtractor().getModel(modelLocation)
: new ModelExtractor().getModel(modelLocation);
graph.importGraphDef(model);
this.session = new Session(graph);
}
/**
* Evaluates a pre-trained tensorflow model (encoded as {@link Graph}). Use the feeds parameter to feed in the
* model input data and fetch-names to specify the output tensors.
*
* Evaluates a pre-trained tensorflow model (encoded as {@link Graph}). Use the feeds
* parameter to feed in the model input data and fetch-names to specify the output
* tensors.
* @param feeds Named map of input tensors.
* @return Returns the computed output tensors. The names of the output tensors is defined by the fetchedNames
* argument
* @return Returns the computed output tensors. The names of the output tensors is
* defined by the fetchedNames argument
*/
@Override
public Map<String, Tensor<?>> apply(Map<String, Tensor<?>> feeds) {
@@ -128,4 +130,5 @@ public class TensorFlowService implements Function<Map<String, Tensor<?>>, Map<S
this.session.close();
}
}
}

36
common/spring-tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/util/AutoCloseables.java
View File

@@ -25,8 +25,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Utilities for AutoCloseable classes.
* Based on the Apache Drill AutoCloseables implementation.
* Utilities for AutoCloseable classes. Based on the Apache Drill AutoCloseables
* implementation.
*/
public final class AutoCloseables {
@@ -45,7 +45,8 @@ public final class AutoCloseables {
}
/**
* Closes all autoCloseables if not null and suppresses exceptions by adding them to t.
* Closes all autoCloseables if not null and suppresses exceptions by adding them to
* t.
* @param t the throwable to add suppressed exception to
* @param autoCloseables the closeables to close
*/
@@ -54,7 +55,8 @@ public final class AutoCloseables {
}
/**
* Closes all autoCloseables if not null and suppresses exceptions by adding them to t.
* Closes all autoCloseables if not null and suppresses exceptions by adding them to
* t.
* @param t the throwable to add suppressed exception to
* @param autoCloseables the closeables to close
*/
@@ -68,7 +70,8 @@ public final class AutoCloseables {
}
/**
* Closes all autoCloseables if not null and suppresses subsequent exceptions if more than one.
* Closes all autoCloseables if not null and suppresses subsequent exceptions if more
* than one.
* @param autoCloseables the closeables to close
*/
public static void close(AutoCloseable... autoCloseables) throws Exception {
@@ -76,7 +79,8 @@ public final class AutoCloseables {
}
/**
* Closes all autoCloseables if not null and suppresses subsequent exceptions if more than one.
* Closes all autoCloseables if not null and suppresses subsequent exceptions if more
* than one.
* @param autoCloseables the closeables to close
*/
public static void close(Iterable<? extends AutoCloseable> autoCloseables) throws Exception {
@@ -102,7 +106,8 @@ public final class AutoCloseables {
}
/**
* Closes all autoCloseables entry values if not null and suppresses subsequent exceptions if more than one.
* Closes all autoCloseables entry values if not null and suppresses subsequent
* exceptions if more than one.
* @param closableMaps the closeables to close
*/
public static void close(Map<?, ? extends AutoCloseable>... closableMaps) throws Exception {
@@ -139,15 +144,14 @@ public final class AutoCloseables {
* @param closeables - array containing auto closeables
*/
public static void closeSilently(AutoCloseable... closeables) {
Arrays.stream(closeables).filter(Objects::nonNull)
.forEach(target -> {
try {
target.close();
}
catch (Exception e) {
LOGGER.warn(String.format("Exception was thrown while closing auto closeable: %s", target), e);
}
});
Arrays.stream(closeables).filter(Objects::nonNull).forEach(target -> {
try {
target.close();
}
catch (Exception e) {
LOGGER.warn(String.format("Exception was thrown while closing auto closeable: %s", target), e);
}
});
}
}

9
common/spring-tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/util/CachedModelExtractor.java
View File

@@ -30,7 +30,9 @@ import org.springframework.core.io.DefaultResourceLoader;
import org.springframework.core.io.Resource;
/**
* Extends the {@link ModelExtractor} to allow keeping a local copy (cache) of the loaded model (protobuf) files.
* Extends the {@link ModelExtractor} to allow keeping a local copy (cache) of the loaded
* model (protobuf) files.
*
* @author Christian Tzolov
*/
public class CachedModelExtractor extends ModelExtractor {
@@ -76,8 +78,8 @@ public class CachedModelExtractor extends ModelExtractor {
String fileName = modelResource.getFilename();
String fragment = modelResource.getURI().getFragment();
File cachedFile = StringUtils.isEmpty(fragment) ? new File(rootFolder, fileName) :
new File(rootFolder, fileName + "_" + fragment);
File cachedFile = StringUtils.isEmpty(fragment) ? new File(rootFolder, fileName)
: new File(rootFolder, fileName + "_" + fragment);
if (cachedFile.exists()) {
logger.info("Load model " + modelResource.toString() + " from cache: " + cacheRootDirectory);
return IOUtils.toByteArray(new FileInputStream(cachedFile));
@@ -104,4 +106,5 @@ public class CachedModelExtractor extends ModelExtractor {
rootFolder.mkdirs();
}
}
}

73
common/spring-tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/util/ModelExtractor.java
View File

@@ -43,15 +43,16 @@ import org.apache.commons.lang3.Validate;
import org.springframework.core.io.DefaultResourceLoader;
import org.springframework.core.io.Resource;
/**
* Extracts a pre-trained (frozen) Tensorflow model URI into byte array. The 'http://', 'file://' and 'classpath://'
* URI schemas are supported.
* Extracts a pre-trained (frozen) Tensorflow model URI into byte array. The 'http://',
* 'file://' and 'classpath://' URI schemas are supported.
*
* Models can be extract either from raw files or form compressed archives. When extracted from an archive the model
* file name can optionally be provided as an URI fragment. For example for resource: http://myarchive.tar.gz#model.pb
* the myarchive.tar.gz is traversed to uncompress and extract the model.pb file as byte array.
* If the file name is not provided as URI fragment then the first file in the archive with extension .pb is extracted.
* Models can be extract either from raw files or form compressed archives. When extracted
* from an archive the model file name can optionally be provided as an URI fragment. For
* example for resource: http://myarchive.tar.gz#model.pb the myarchive.tar.gz is
* traversed to uncompress and extract the model.pb file as byte array. If the file name
* is not provided as URI fragment then the first file in the archive with extension .pb
* is extracted.
*
* @author Christian Tzolov
*/
@@ -60,9 +61,10 @@ public class ModelExtractor {
private static final String DEFAULT_FROZEN_GRAPH_FILE_EXTENSION = ".pb";
/**
* When an archive resource if referred, but no fragment URI is provided (to specify the target file name in
* the archive) then the extractor selects the first file in the archive with the extension that match
* the frozenGraphFileExtension (defaults to .pb).
* When an archive resource if referred, but no fragment URI is provided (to specify
* the target file name in the archive) then the extractor selects the first file in
* the archive with the extension that match the frozenGraphFileExtension (defaults to
* .pb).
*/
public final String frozenGraphFileExtension;
@@ -89,11 +91,12 @@ public class ModelExtractor {
String compressor = archiveCompressor[1];
String fragment = modelResource.getURI().getFragment();
if (StringUtils.isNotBlank(compressor)) {
try (CompressorInputStream cis = new CompressorStreamFactory().createCompressorInputStream(compressor, bi)) {
try (CompressorInputStream cis = new CompressorStreamFactory().createCompressorInputStream(compressor,
bi)) {
if (StringUtils.isNotBlank(archive)) {
try (ArchiveInputStream ais = new ArchiveStreamFactory().createArchiveInputStream(archive, cis)) {
try (ArchiveInputStream ais = new ArchiveStreamFactory().createArchiveInputStream(archive,
cis)) {
// Compressor fromMemory Archive
return findInArchiveStream(fragment, ais);
}
@@ -119,22 +122,24 @@ public class ModelExtractor {
}
/**
* Traverses the Archive to find either an entry that matches the modelFileNameInArchive name (if not empty) or
* and entry that ends in .pb if the modelFileNameInArchive is empty.
*
* @param modelFileNameInArchive Optional name of the archive entry that represents the frozen model file. If empty
* the archive will be searched for the first entry that ends in .pb
* Traverses the Archive to find either an entry that matches the
* modelFileNameInArchive name (if not empty) or and entry that ends in .pb if the
* modelFileNameInArchive is empty.
* @param modelFileNameInArchive Optional name of the archive entry that represents
* the frozen model file. If empty the archive will be searched for the first entry
* that ends in .pb
* @param archive Archive stream to be traversed
*
*/
private byte[] findInArchiveStream(String modelFileNameInArchive, ArchiveInputStream archive) throws IOException {
ArchiveEntry entry;
while ((entry = archive.getNextEntry()) != null) {
//System.out.println(entry.getName() + " : " + entry.isDirectory());
// System.out.println(entry.getName() + " : " + entry.isDirectory());
if (archive.canReadEntryData(entry) && !entry.isDirectory()) {
if ((StringUtils.isNotBlank(modelFileNameInArchive) && entry.getName().endsWith(modelFileNameInArchive)) ||
(!StringUtils.isNotBlank(modelFileNameInArchive) && entry.getName().endsWith(this.frozenGraphFileExtension))) {
if ((StringUtils.isNotBlank(modelFileNameInArchive) && entry.getName().endsWith(modelFileNameInArchive))
|| (!StringUtils.isNotBlank(modelFileNameInArchive)
&& entry.getName().endsWith(this.frozenGraphFileExtension))) {
return IOUtils.toByteArray(archive);
}
}
@@ -144,22 +149,20 @@ public class ModelExtractor {
/**
* Detect the Archive and the Compressor from the file extension.
*
* @param fileName File name with extension.
* @return Returns a tuple of the detected (Archive, Compressor). Null stands for not available
* archive or detector. The (null, null) response stands for no Archive or Compressor discovered.
* @return Returns a tuple of the detected (Archive, Compressor). Null stands for not
* available archive or detector. The (null, null) response stands for no Archive or
* Compressor discovered.
*/
private String[] detectArchiveAndCompressor(String fileName) {
String normalizedFileName = fileName.trim().toLowerCase();
if (normalizedFileName.endsWith(".tar.gz")
|| normalizedFileName.endsWith(".tgz")
if (normalizedFileName.endsWith(".tar.gz") || normalizedFileName.endsWith(".tgz")
|| normalizedFileName.endsWith(".taz")) {
return new String[] { ArchiveStreamFactory.TAR, CompressorStreamFactory.GZIP };
}
else if (normalizedFileName.endsWith(".tar.bz2")
|| normalizedFileName.endsWith(".tbz2")
else if (normalizedFileName.endsWith(".tar.bz2") || normalizedFileName.endsWith(".tbz2")
|| normalizedFileName.endsWith(".tbz")) {
return new String[] { ArchiveStreamFactory.TAR, CompressorStreamFactory.BZIP2 };
}
@@ -175,8 +178,7 @@ public class ModelExtractor {
else if (normalizedFileName.endsWith(".gzip")) {
return new String[] { null, CompressorStreamFactory.GZIP };
}
else if (normalizedFileName.endsWith(".bz2")
|| normalizedFileName.endsWith(".bz")) {
else if (normalizedFileName.endsWith(".bz2") || normalizedFileName.endsWith(".bz")) {
return new String[] { null, CompressorStreamFactory.BZIP2 };
}
@@ -190,7 +192,9 @@ public class ModelExtractor {
private Optional<String> findArchive(String normalizedFileName) {
return new ArchiveStreamFactory().getInputStreamArchiveNames()
.stream().filter(arch -> normalizedFileName.endsWith("." + arch)).findFirst();
.stream()
.filter(arch -> normalizedFileName.endsWith("." + arch))
.findFirst();
}
private boolean hasCompressor(String normalizedFileName) {
@@ -199,7 +203,9 @@ public class ModelExtractor {
private Optional<String> findCompressor(String normalizedFileName) {
return new CompressorStreamFactory().getInputStreamCompressorNames()
.stream().filter(compressor -> normalizedFileName.endsWith("." + compressor)).findFirst();
.stream()
.filter(compressor -> normalizedFileName.endsWith("." + compressor))
.findFirst();
}
static {
@@ -219,8 +225,7 @@ public class ModelExtractor {
public void checkServerTrusted(X509Certificate[] certs, String authType) {
}
}
};
} };
// Install the all-trusting trust manager
SSLContext sc = SSLContext.getInstance("SSL");

50
common/spring-tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/EnrichFromMemory.java
View File

@@ -28,42 +28,40 @@ import org.tensorflow.Tensor;
public class EnrichFromMemory implements AutoCloseable {
private final GraphRunner graph1;
private final GraphRunner graph2;
private final GraphRunner graph3;
public EnrichFromMemory() {
this.graph1 = new GraphRunner("x1", "y1")
.withGraphDefinition(tf -> tf.withName("y1").math.mul(
tf.withName("x1").placeholder(Integer.class),
tf.constant(10)));
this.graph1 = new GraphRunner("x1", "y1").withGraphDefinition(
tf -> tf.withName("y1").math.mul(tf.withName("x1").placeholder(Integer.class), tf.constant(10)));
this.graph2 = new GraphRunner("x2", "y2")
.withGraphDefinition(tf -> tf.withName("y2").math.mul(
tf.withName("x2").placeholder(Integer.class),
tf.constant(20)));
this.graph2 = new GraphRunner("x2", "y2").withGraphDefinition(
tf -> tf.withName("y2").math.mul(tf.withName("x2").placeholder(Integer.class), tf.constant(20)));
this.graph3 = new GraphRunner(Arrays.asList("x31", "x32"), Arrays.asList("y3"))
.withGraphDefinition(tf -> tf.withName("y3").math.add(
tf.withName("x31").placeholder(Integer.class),
tf.withName("x32").placeholder(Integer.class)));
.withGraphDefinition(tf -> tf.withName("y3").math.add(tf.withName("x31").placeholder(Integer.class),
tf.withName("x32").placeholder(Integer.class)));
}
public int compute(Integer input) {
try (
Tensor x = Tensor.create(input);
GraphRunnerMemory memory = new GraphRunnerMemory();
) {
try (Tensor x = Tensor.create(input); GraphRunnerMemory memory = new GraphRunnerMemory();) {
Map<String, Tensor<?>> result =
this.graph1.andThen(memory)
.andThen(graph2).andThen(memory)
.andThen(Functions.enrichFromMemory(memory, "y1")) // retrieves the graph1's y1 output and adds it as a parameter with the same name
.andThen(Functions.rename(
"y1", "x31", // renames the input y1 into x31
"y2", "x32" // renames the input y2 into x32
))
.andThen(graph3).andThen(memory)
.apply(Collections.singletonMap("x", x));
Map<String, Tensor<?>> result = this.graph1.andThen(memory)
.andThen(graph2)
.andThen(memory)
.andThen(Functions.enrichFromMemory(memory, "y1")) // retrieves the
// graph1's y1 output
// and adds it as a
// parameter with the
// same name
.andThen(Functions.rename("y1", "x31", // renames the input y1 into x31
"y2", "x32" // renames the input y2 into x32
))
.andThen(graph3)
.andThen(memory)
.apply(Collections.singletonMap("x", x));
memory.getTensorMap().entrySet().forEach(e -> System.out.println(" " + e));
@@ -85,5 +83,5 @@ public class EnrichFromMemory implements AutoCloseable {
}
}
}
}
}

18
common/spring-tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/FunctionComposition.java
View File

@@ -35,17 +35,11 @@ public final class FunctionComposition {
// y1 = x1 * 2 , where x1 == x
// y2 = x2 + 20 , where x2 == y1 and y = y2
public static void main(String[] args) {
try (
GraphRunner graph1 = new GraphRunner("x1", "y1")
.withGraphDefinition(tf -> tf.withName("y1").math.mul(
tf.withName("x1").placeholder(Integer.class),
tf.constant(2)));
GraphRunner graph2 = new GraphRunner("x2", "y2")
.withGraphDefinition(tf -> tf.withName("y2").math.add(
tf.withName("x2").placeholder(Integer.class),
tf.constant(20)));
Tensor x = Tensor.create(10);
) {
try (GraphRunner graph1 = new GraphRunner("x1", "y1").withGraphDefinition(
tf -> tf.withName("y1").math.mul(tf.withName("x1").placeholder(Integer.class), tf.constant(2)));
GraphRunner graph2 = new GraphRunner("x2", "y2").withGraphDefinition(tf -> tf.withName("y2").math
.add(tf.withName("x2").placeholder(Integer.class), tf.constant(20)));
Tensor x = Tensor.create(10);) {
Map<String, Tensor<?>> result = graph1.andThen(graph2).apply(Collections.singletonMap("x", x));
@@ -54,5 +48,5 @@ public final class FunctionComposition {
}
}
}
}

36
common/spring-tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/FunctionCompositionMultipleInputsOutputs.java
View File

@@ -38,30 +38,24 @@ public final class FunctionCompositionMultipleInputsOutputs {
// y12 = x1 * 3 , where x1 == x
// y2 = x21 + x22 , where x21 == y11, x22 == y12 and y == y2
public static void main(String[] args) {
try (
GraphRunner graph1 = new GraphRunner(Arrays.asList("x1"), Arrays.asList("y11", "y12"))
.withGraphDefinition(tf -> {
Placeholder<Integer> x1 = tf.withName("x1").placeholder(Integer.class);
tf.withName("y11").math.mul(x1, tf.constant(2));
tf.withName("y12").math.mul(x1, tf.constant(3));
});
try (GraphRunner graph1 = new GraphRunner(Arrays.asList("x1"), Arrays.asList("y11", "y12"))
.withGraphDefinition(tf -> {
Placeholder<Integer> x1 = tf.withName("x1").placeholder(Integer.class);
tf.withName("y11").math.mul(x1, tf.constant(2));
tf.withName("y12").math.mul(x1, tf.constant(3));
});
GraphRunner graph2 = new GraphRunner(Arrays.asList("x21", "x22"), Arrays.asList("y2"))
.withGraphDefinition(tf -> tf.withName("y2").math.add(
tf.withName("x21").placeholder(Integer.class),
tf.withName("x22").placeholder(Integer.class)));
Tensor x = Tensor.create(10);
) {
.withGraphDefinition(tf -> tf.withName("y2").math.add(tf.withName("x21").placeholder(Integer.class),
tf.withName("x22").placeholder(Integer.class)));
Tensor x = Tensor.create(10);) {
Map<String, Tensor<?>> result = graph1
.andThen(Functions.rename(
"y11", "x21",
"y12", "x22"
))
.andThen(graph2)
.apply(Collections.singletonMap("x", x));
Map<String, Tensor<?>> result = graph1.andThen(Functions.rename("y11", "x21", "y12", "x22"))
.andThen(graph2)
.apply(Collections.singletonMap("x", x));
System.out.println("Result is: " + result.get("y2").intValue()); // Result is: 50
System.out.println("Result is: " + result.get("y2").intValue()); // Result is:
// 50
}
}
}
}

28
common/spring-tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/ReleaseTensorParameters.java
View File

@@ -27,31 +27,25 @@ import org.tensorflow.Tensor;
public class ReleaseTensorParameters implements AutoCloseable {
private final GraphRunner graph1;
private final GraphRunner graph2;
public ReleaseTensorParameters() {
this.graph1 = new GraphRunner("x1", "y1")
.withGraphDefinition(tf -> tf.withName("y1").math.mul(
tf.withName("x1").placeholder(Integer.class),
tf.constant(2)));
this.graph1 = new GraphRunner("x1", "y1").withGraphDefinition(
tf -> tf.withName("y1").math.mul(tf.withName("x1").placeholder(Integer.class), tf.constant(2)));
this.graph2 = new GraphRunner("x2", "y2")
.withGraphDefinition(tf -> tf.withName("y2").math.add(
tf.withName("x2").placeholder(Integer.class),
tf.constant(20)));
this.graph2 = new GraphRunner("x2", "y2").withGraphDefinition(
tf -> tf.withName("y2").math.add(tf.withName("x2").placeholder(Integer.class), tf.constant(20)));
}
// y = (x * 2) + 20
public int compute(Integer input) {
try (
Tensor x = Tensor.create(input);
GraphRunnerMemory memory = new GraphRunnerMemory();
) {
try (Tensor x = Tensor.create(input); GraphRunnerMemory memory = new GraphRunnerMemory();) {
Map<String, Tensor<?>> result =
this.graph1.andThen(memory)
.andThen(graph2).andThen(memory)
.apply(Collections.singletonMap("x", x));
Map<String, Tensor<?>> result = this.graph1.andThen(memory)
.andThen(graph2)
.andThen(memory)
.apply(Collections.singletonMap("x", x));
memory.getTensorMap().entrySet().forEach(e -> System.out.println(" " + e));
@@ -73,5 +67,5 @@ public class ReleaseTensorParameters implements AutoCloseable {
}
}
}
}
}

2
common/spring-twitter-common/src/main/java/org/springframework/cloud/fn/common/twitter/Cursor.java
View File

@@ -20,6 +20,7 @@ package org.springframework.cloud.fn.common.twitter;
* @author Christian Tzolov
*/
public class Cursor {
private long cursor = -1;
public long getCursor() {
@@ -34,4 +35,5 @@ public class Cursor {
public String toString() {
return "Cursor{cursor=" + cursor + '}';
}
}

32
common/spring-twitter-common/src/main/java/org/springframework/cloud/fn/common/twitter/TwitterConnectionConfiguration.java
View File

@@ -40,7 +40,6 @@ import org.springframework.messaging.support.MessageBuilder;
import org.springframework.util.MimeTypeUtils;
/**
*
* @author Christian Tzolov
*/
@Configuration
@@ -67,13 +66,12 @@ public class TwitterConnectionConfiguration {
@Bean
public Function<TwitterConnectionProperties, ConfigurationBuilder> toConfigurationBuilder() {
return properties -> new ConfigurationBuilder()
.setJSONStoreEnabled(properties.isRawJson())
.setDebugEnabled(properties.isDebugEnabled())
.setOAuthConsumerKey(properties.getConsumerKey())
.setOAuthConsumerSecret(properties.getConsumerSecret())
.setOAuthAccessToken(properties.getAccessToken())
.setOAuthAccessTokenSecret(properties.getAccessTokenSecret());
return properties -> new ConfigurationBuilder().setJSONStoreEnabled(properties.isRawJson())
.setDebugEnabled(properties.isDebugEnabled())
.setOAuthConsumerKey(properties.getConsumerKey())
.setOAuthConsumerSecret(properties.getConsumerSecret())
.setOAuthAccessToken(properties.getAccessToken())
.setOAuthAccessTokenSecret(properties.getAccessTokenSecret());
}
@Bean
@@ -82,10 +80,9 @@ public class TwitterConnectionConfiguration {
try {
String json = mapper.writeValueAsString(objects);
return MessageBuilder
.withPayload(json.getBytes())
.setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON_VALUE)
.build();
return MessageBuilder.withPayload(json.getBytes())
.setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON_VALUE)
.build();
}
catch (JsonProcessingException e) {
logger.error("Status to JSON conversion error!", e);
@@ -95,12 +92,12 @@ public class TwitterConnectionConfiguration {
}
/**
* Retrieves the raw JSON form of the provided object.
* Retrieves the raw JSON form of the provided object.
*
* Note that raw JSON forms can be retrieved only from the same thread invoked the last method
* call and will become inaccessible once another method call.
*
* @return Function that can retrieve the raw JSON object from the objects returned by the Twitter4J's APIs.
* Note that raw JSON forms can be retrieved only from the same thread invoked the
* last method call and will become inaccessible once another method call.
* @return Function that can retrieve the raw JSON object from the objects returned by
* the Twitter4J's APIs.
*/
@Bean
public Function<Object, Object> rawJsonExtractor() {
@@ -124,4 +121,5 @@ public class TwitterConnectionConfiguration {
Function<Object, Object> rawJsonExtractor, Function<Object, Message<byte[]>> json) {
return list -> (properties.isRawJson()) ? rawJsonExtractor.andThen(json).apply(list) : json.apply(list);
}
}

6
common/spring-twitter-common/src/main/java/org/springframework/cloud/fn/common/twitter/TwitterConnectionProperties.java
View File

@@ -21,7 +21,6 @@ import jakarta.validation.constraints.NotEmpty;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.validation.annotation.Validated;
/**
* @author Christian Tzolov
*/
@@ -60,8 +59,8 @@ public class TwitterConnectionProperties {
/**
* Enable caching the original (raw) JSON objects as returned by the Twitter APIs.
* When set to False the result will use the Twitter4J's json representations.
* When set to True the result will use the original Twitter APISs json representations.
* When set to False the result will use the Twitter4J's json representations. When
* set to True the result will use the original Twitter APISs json representations.
*/
private boolean rawJson = true;
@@ -112,4 +111,5 @@ public class TwitterConnectionProperties {
public void setRawJson(boolean rawJson) {
this.rawJson = rawJson;
}
}

7
common/spring-twitter-common/src/main/java/org/springframework/cloud/fn/common/twitter/util/TwitterTestUtils.java
View File

@@ -16,7 +16,6 @@
package org.springframework.cloud.fn.common.twitter.util;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.function.Function;
@@ -52,8 +51,10 @@ public class TwitterTestUtils {
/**
* Load Spring Resource as String.
* @param resourcePath Resource path (accepts file:// , classpath:// and http:// uri schemas)
* @return Returns text (UTF8) representation of the resource pointed by the resourcePath
* @param resourcePath Resource path (accepts file:// , classpath:// and http:// uri
* schemas)
* @return Returns text (UTF8) representation of the resource pointed by the
* resourcePath
*/
public static String asString(String resourcePath) {
try {

10
common/spring-xmpp-common/src/main/java/org/springframework/cloud/fn/common/xmpp/XmppConnectionFactoryConfiguration.java
View File

@@ -33,7 +33,8 @@ public class XmppConnectionFactoryConfiguration {
@Bean
@ConditionalOnMissingBean
public XmppConnectionFactoryBean xmppConnectionFactoryBean(XmppConnectionFactoryProperties properties) throws XmppStringprepException {
public XmppConnectionFactoryBean xmppConnectionFactoryBean(XmppConnectionFactoryProperties properties)
throws XmppStringprepException {
XmppConnectionFactoryBean xmppConnectionFactoryBean = new XmppConnectionFactoryBean();
xmppConnectionFactoryBean.setSubscriptionMode(properties.getSubscriptionMode());
@@ -48,11 +49,12 @@ public class XmppConnectionFactoryConfiguration {
if (StringUtils.hasText(properties.getServiceName())) {
builder.setUsernameAndPassword(properties.getUser(), properties.getPassword())
.setXmppDomain(properties.getServiceName());
.setXmppDomain(properties.getServiceName());
}
else {
builder.setUsernameAndPassword(XmppStringUtils.parseLocalpart(properties.getUser()), properties.getPassword())
.setXmppDomain(properties.getUser());
builder
.setUsernameAndPassword(XmppStringUtils.parseLocalpart(properties.getUser()), properties.getPassword())
.setXmppDomain(properties.getUser());
}
xmppConnectionFactoryBean.setConnectionConfiguration(builder.build());

8
common/spring-xmpp-common/src/main/java/org/springframework/cloud/fn/common/xmpp/XmppConnectionFactoryProperties.java
View File

@@ -24,7 +24,6 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.validation.annotation.Validated;
/**
*
* @author Daniel Frey
* @since 4.0.0
*/
@@ -33,8 +32,8 @@ import org.springframework.validation.annotation.Validated;
public class XmppConnectionFactoryProperties {
/**
* The Resource to bind to on the XMPP Host.
* - Can be empty, server will generate one if not set
* The Resource to bind to on the XMPP Host. - Can be empty, server will generate one
* if not set
*/
private String resource;
@@ -59,8 +58,7 @@ public class XmppConnectionFactoryProperties {
private String host;
/**
* Port for connecting to the host.
* - Default Client Port: 5222
* Port for connecting to the host. - Default Client Port: 5222
*/
private int port = 5222;

107
consumer/spring-analytics-consumer/src/main/java/org/springframework/cloud/fn/consumer/analytics/AnalyticsConsumerConfiguration.java
View File

@@ -62,20 +62,24 @@ public class AnalyticsConsumerConfiguration {
private final Map<Meter.Id, AtomicLong> gaugeValues = new ConcurrentHashMap<>();
@Bean(name = "analyticsConsumer")
public Consumer<Message<?>> analyticsConsumer(AnalyticsConsumerProperties properties, MeterRegistry[] meterRegistries,
public Consumer<Message<?>> analyticsConsumer(AnalyticsConsumerProperties properties,
MeterRegistry[] meterRegistries,
@Lazy @Qualifier(SpelExpressionConverterConfiguration.INTEGRATION_EVALUATION_CONTEXT) EvaluationContext context) {
// If the CompositeMeterRegistry is present the it already contains all non-composite registries.
// In this case we override the input meterRegistries to use the CompositeMeterRegistry only.
// If the CompositeMeterRegistry is present the it already contains all
// non-composite registries.
// In this case we override the input meterRegistries to use the
// CompositeMeterRegistry only.
final MeterRegistry[] finalMeterRegistries = Stream.of(meterRegistries)
.filter(CompositeMeterRegistry.class::isInstance)
.findFirst()
.map(meterRegistry -> new MeterRegistry[] { meterRegistry })
.orElse(meterRegistries);
.filter(CompositeMeterRegistry.class::isInstance)
.findFirst()
.map(meterRegistry -> new MeterRegistry[] { meterRegistry })
.orElse(meterRegistries);
return message -> {
CharSequence meterNameRaw = properties.getComputedNameExpression().getValue(context, message, CharSequence.class);
CharSequence meterNameRaw = properties.getComputedNameExpression()
.getValue(context, message, CharSequence.class);
String meterName = StringUtils.isEmpty(meterNameRaw) ? "empty" : meterNameRaw.toString();
// All fixed tags together are passed with every meter update.
@@ -87,46 +91,52 @@ public class AnalyticsConsumerConfiguration {
// Tag Expressions
if (properties.getTag().getExpression() != null) {
Map<String, List<Tag>> groupedTags = properties.getTag().getExpression().entrySet().stream()
// maps a <name, expr> pair into [<name, expr#val_1>, ... <name, expr#val_N>] Tag array.
.map(namedExpression ->
toList(namedExpression.getValue().getValue(context, message)).stream()
.map(tagValue -> Tag.of(namedExpression.getKey(), tagValue))
.collect(Collectors.toList())).flatMap(List::stream)
.collect(Collectors.groupingBy(Tag::getKey, Collectors.toList()));
Map<String, List<Tag>> groupedTags = properties.getTag()
.getExpression()
.entrySet()
.stream()
// maps a <name, expr> pair into [<name, expr#val_1>, ... <name,
// expr#val_N>] Tag array.
.map(namedExpression -> toList(namedExpression.getValue().getValue(context, message)).stream()
.map(tagValue -> Tag.of(namedExpression.getKey(), tagValue))
.collect(Collectors.toList()))
.flatMap(List::stream)
.collect(Collectors.groupingBy(Tag::getKey, Collectors.toList()));
allGroupedTags.putAll(groupedTags);
}
this.recordMetrics(finalMeterRegistries, meterName, fixedTags, allGroupedTags, amount, properties.getMeterType());
this.recordMetrics(finalMeterRegistries, meterName, fixedTags, allGroupedTags, amount,
properties.getMeterType());
};
}
/**
* Converts a key/value Map into Tag(key,value) list. Filters out the empty key/value pairs.
*
* Converts a key/value Map into Tag(key,value) list. Filters out the empty key/value
* pairs.
* @param keyValueMap key/value map to convert into tags.
* @return Returns Tags list representing every non-empty key/value pair.
*/
protected Tags toTags(Map<String, String> keyValueMap) {
return CollectionUtils.isEmpty(keyValueMap) ? Tags.empty() :
Tags.of(keyValueMap.entrySet().stream()
.filter(e -> StringUtils.hasText(e.getKey()) && StringUtils.hasText(e.getValue()))
.map(e -> Tag.of(e.getKey(), e.getValue()))
.collect(Collectors.toList()));
return CollectionUtils.isEmpty(keyValueMap) ? Tags.empty()
: Tags.of(keyValueMap.entrySet()
.stream()
.filter(e -> StringUtils.hasText(e.getKey()) && StringUtils.hasText(e.getValue()))
.map(e -> Tag.of(e.getKey(), e.getValue()))
.collect(Collectors.toList()));
}
/**
* Converts the input value into an list of values. If the value is not a collection/array type the result
* is a single element list. For collection/array input value the result is the list of stringified content of
* this collection.
*
* Converts the input value into an list of values. If the value is not a
* collection/array type the result is a single element list. For collection/array
* input value the result is the list of stringified content of this collection.
* @param value input value can be array, collection or single value.
* @return Returns value list.
*/
protected List<String> toList(Object value) {
if (value == null) {
// Ensure that the tag is present in the meter metrics, even if empty.
// TSDB as Prometheus do not tolerate same meters to have different tags signatures.
// TSDB as Prometheus do not tolerate same meters to have different tags
// signatures.
return Collections.singletonList(UNAVAILABLE_TAG);
}
@@ -135,10 +145,10 @@ public class AnalyticsConsumerConfiguration {
Collection<?> valueCollection = (value instanceof Collection) ? (Collection<?>) value
: Arrays.asList(ObjectUtils.toObjectArray(value));
List<String> list = valueCollection.stream()
.filter(Objects::nonNull)
.map(Object::toString)
.filter(StringUtils::hasText)
.collect(Collectors.toList());
.filter(Objects::nonNull)
.map(Object::toString)
.filter(StringUtils::hasText)
.collect(Collectors.toList());
return CollectionUtils.isEmpty(list) ? Collections.singletonList(UNAVAILABLE_TAG) : list;
}
else {
@@ -149,21 +159,18 @@ public class AnalyticsConsumerConfiguration {
private void recordMetrics(MeterRegistry[] meterRegistries, String meterName, Tags fixedTags,
Map<String, List<Tag>> groupedTags, double amount, AnalyticsConsumerProperties.MeterType meterType) {
if (!CollectionUtils.isEmpty(groupedTags)) {
groupedTags.values().stream().map(List::size).max(Integer::compareTo).ifPresent(
max -> {
for (int i = 0; i < max; i++) {
Tags currentTags = Tags.of(fixedTags);
for (Map.Entry<String, List<Tag>> e : groupedTags.entrySet()) {
currentTags = (e.getValue().size() > i) ?
currentTags.and(e.getValue().get(i)) :
currentTags.and(Tags.of(e.getKey(), ""));
}
// Update the meterName for every configured MaterRegistry.
record(meterRegistries, meterName, currentTags, amount, meterType);
}
groupedTags.values().stream().map(List::size).max(Integer::compareTo).ifPresent(max -> {
for (int i = 0; i < max; i++) {
Tags currentTags = Tags.of(fixedTags);
for (Map.Entry<String, List<Tag>> e : groupedTags.entrySet()) {
currentTags = (e.getValue().size() > i) ? currentTags.and(e.getValue().get(i))
: currentTags.and(Tags.of(e.getKey(), ""));
}
);
// Update the meterName for every configured MaterRegistry.
record(meterRegistries, meterName, currentTags, amount, meterType);
}
});
}
else {
// Update the meterName for every configured MaterRegistry.
@@ -171,8 +178,8 @@ public class AnalyticsConsumerConfiguration {
}
}
private void record(MeterRegistry[] meterRegistries, String meterName,
Iterable<Tag> tags, double meterAmount, AnalyticsConsumerProperties.MeterType meterType) {
private void record(MeterRegistry[] meterRegistries, String meterName, Iterable<Tag> tags, double meterAmount,
AnalyticsConsumerProperties.MeterType meterType) {
for (MeterRegistry meterRegistry : meterRegistries) {
if (meterType == AnalyticsConsumerProperties.MeterType.gauge) {
@@ -197,8 +204,7 @@ public class AnalyticsConsumerConfiguration {
}
private boolean isMeterRegistryContainsGauge(MeterRegistry meterRegistry, Meter.Id gaugeId) {
return meterRegistry.find(gaugeId.getName()).gauges().stream()
.anyMatch(gauge -> gauge.getId().equals(gaugeId));
return meterRegistry.find(gaugeId.getName()).gauges().stream().anyMatch(gauge -> gauge.getId().equals(gaugeId));
}
@Bean
@@ -206,4 +212,5 @@ public class AnalyticsConsumerConfiguration {
public SimpleMeterRegistry simpleMeterRegistry() {
return new SimpleMeterRegistry();
}
}

62
consumer/spring-analytics-consumer/src/main/java/org/springframework/cloud/fn/consumer/analytics/AnalyticsConsumerProperties.java
View File

@@ -34,13 +34,20 @@ import org.springframework.validation.annotation.Validated;
public class AnalyticsConsumerProperties {
enum MeterType {
/** Uses the Micrometer Counter meter type. It accumulates intermediate counts toward the point where
* the data is sent to the metrics backend.*/
/**
* Uses the Micrometer Counter meter type. It accumulates intermediate counts
* toward the point where the data is sent to the metrics backend.
*/
counter,
/** Uses the Micrometer Gauge meter type. Gauges sample the input time series. Any intermediate values set on
* a gauge are lost by the time the gauge value is reported to a metrics backend.
* TIP: Never gauge something you can count with a Counter!*/
/**
* Uses the Micrometer Gauge meter type. Gauges sample the input time series. Any
* intermediate values set on a gauge are lost by the time the gauge value is
* reported to a metrics backend. TIP: Never gauge something you can count with a
* Counter!
*/
gauge
}
/**
@@ -49,27 +56,27 @@ public class AnalyticsConsumerProperties {
private MeterType meterType = MeterType.counter;
/**
* The name of the output metrics.
* The 'name' and 'nameExpression' are mutually exclusive. Only one of them can be set.
* The name of the output metrics. The 'name' and 'nameExpression' are mutually
* exclusive. Only one of them can be set.
*/
private String name;
/**
* A SpEL expression to compute the output metrics name from the input message.
* The 'name' and 'nameExpression' are mutually exclusive. Only one of them can be set.
* A SpEL expression to compute the output metrics name from the input message. The
* 'name' and 'nameExpression' are mutually exclusive. Only one of them can be set.
*/
private Expression nameExpression;
/**
* If neither `name` nor `nameExpression` is set the name of the output metrics defaults to the
* Spring application name.
* If neither `name` nor `nameExpression` is set the name of the output metrics
* defaults to the Spring application name.
*/
@Value("${spring.application.name:analytics}")
private String defaultName;
/**
* A SpEL expression to compute the output metrics value (e.g. amount).
* It defaults to 1.0
* A SpEL expression to compute the output metrics value (e.g. amount). It defaults to
* 1.0
*/
private Expression amountExpression;
@@ -132,20 +139,18 @@ public class AnalyticsConsumerProperties {
@Override
public String toString() {
return "AnalyticsFunctionProperties{" +
"defaultName='" + defaultName + '\'' +
", name=" + name +
", tag=" + tag +
'}';
return "AnalyticsFunctionProperties{" + "defaultName='" + defaultName + '\'' + ", name=" + name + ", tag=" + tag
+ '}';
}
public static class MetricsTag {
/**
* DEPRECATED: Please use the analytics.tag.expression with literal SpEL expression.
* DEPRECATED: Please use the analytics.tag.expression with literal SpEL
* expression.
*
* Custom, fixed Tags. Those tags have constant values, created once and then sent along with every
* published metrics. The convention to define a fixed Tags is:
* Custom, fixed Tags. Those tags have constant values, created once and then sent
* along with every published metrics. The convention to define a fixed Tags is:
* <code>
* analytics.tag.fixed.[tag-name]=[tag-value]
* </code>
@@ -154,10 +159,10 @@ public class AnalyticsConsumerProperties {
private Map<String, String> fixed;
/**
* Computes tags from SpEL expression.
* Single SpEL expression can produce an array of values, which in turn means distinct name/value tags.
* Every name/value tag will produce a separate meter increment.
* Tag expression format is: analytics.tag.expression.[tag-name]=[SpEL expression]
* Computes tags from SpEL expression. Single SpEL expression can produce an array
* of values, which in turn means distinct name/value tags. Every name/value tag
* will produce a separate meter increment. Tag expression format is:
* analytics.tag.expression.[tag-name]=[SpEL expression]
*/
private Map<String, Expression> expression;
@@ -179,10 +184,9 @@ public class AnalyticsConsumerProperties {
@Override
public String toString() {
return "MetricsTag{" +
"fixed=" + fixed +
", expression=" + expression +
'}';
return "MetricsTag{" + "fixed=" + fixed + ", expression=" + expression + '}';
}
}
}

2
consumer/spring-analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/AnalyticsConsumerParentTest.java
View File

@@ -44,5 +44,7 @@ public class AnalyticsConsumerParentTest {
@SpringBootApplication
static class AnalyticsConsumerTestApplication {
}
}

8
consumer/spring-analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/CountWithAmountTest.java
View File

@@ -26,11 +26,8 @@ import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Christian Tzolov
*/
@TestPropertySource(properties = {
"analytics.name=counter666",
"analytics.tag.expression.foo='bar'",
"analytics.amount-expression=payload.length()"
})
@TestPropertySource(properties = { "analytics.name=counter666", "analytics.tag.expression.foo='bar'",
"analytics.amount-expression=payload.length()" })
class CountWithAmountTest extends AnalyticsConsumerParentTest {
@Test
@@ -40,4 +37,5 @@ class CountWithAmountTest extends AnalyticsConsumerParentTest {
analyticsConsumer.accept(new GenericMessage<>(message));
assertThat(meterRegistry.find("counter666").counter().count()).isEqualTo(messageSize);
}
}

11
consumer/spring-analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/EmptyTagsTests.java
View File

@@ -28,12 +28,9 @@ import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Christian Tzolov
*/
@TestPropertySource(properties = {
"analytics.name=counter666",
"analytics.tag.fixed.foo=",
@TestPropertySource(properties = { "analytics.name=counter666", "analytics.tag.fixed.foo=",
"analytics.tag.expression.tag666=#jsonPath(payload,'$..noField')",
"analytics.tag.expression.test=#jsonPath(payload,'$..test')"
})
"analytics.tag.expression.test=#jsonPath(payload,'$..test')" })
class EmptyTagsTests extends AnalyticsConsumerParentTest {
@Test
@@ -46,10 +43,12 @@ class EmptyTagsTests extends AnalyticsConsumerParentTest {
Collection<Counter> expressionTagsCounters = meterRegistry.find("counter666").tagKeys("tag666").counters();
assertThat(expressionTagsCounters.size()).isEqualTo(1);
assertThat(meterRegistry.find("counter666").meter().getId().getTag("tag666")).isEqualTo(AnalyticsConsumerConfiguration.UNAVAILABLE_TAG);
assertThat(meterRegistry.find("counter666").meter().getId().getTag("tag666"))
.isEqualTo(AnalyticsConsumerConfiguration.UNAVAILABLE_TAG);
Collection<Counter> testExpTagsCounters = meterRegistry.find("counter666").tagKeys("test").counters();
assertThat(testExpTagsCounters.size()).isEqualTo(1);
assertThat(meterRegistry.find("counter666").meter().getId().getTag("test")).isEqualTo("Bar");
}
}

5
consumer/spring-analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/ExpressionCounterNameTests.java
View File

@@ -28,9 +28,7 @@ import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Christian Tzolov
*/
@TestPropertySource(properties = {
"analytics.name-expression=payload"
})
@TestPropertySource(properties = { "analytics.name-expression=payload" })
public class ExpressionCounterNameTests extends AnalyticsConsumerParentTest {
@Test
@@ -38,4 +36,5 @@ public class ExpressionCounterNameTests extends AnalyticsConsumerParentTest {
IntStream.range(0, 13).forEach(i -> analyticsConsumer.accept(new GenericMessage<>("hello")));
assertThat(meterRegistry.find("hello").counter().count()).isEqualTo(13.0);
}
}

13
consumer/spring-analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/FixedTagsTests.java
View File

@@ -30,22 +30,21 @@ import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Christian Tzolov
*/
@TestPropertySource(properties = {
"analytics.name=counter666",
"analytics.tag.fixed.foo=bar",
"analytics.tag.fixed.gork=bork"
})
@TestPropertySource(
properties = { "analytics.name=counter666", "analytics.tag.fixed.foo=bar", "analytics.tag.fixed.gork=bork" })
public class FixedTagsTests extends AnalyticsConsumerParentTest {
@Test
void testAnalyticsSink() {
IntStream.range(0, 13).forEach(i -> analyticsConsumer.accept(new GenericMessage<>("hello")));
Meter counterMeter = meterRegistry.find("counter666").meter();
assertThat(StreamSupport.stream(counterMeter.measure().spliterator(), false)
.mapToDouble(m -> m.getValue()).sum()).isEqualTo(13.0);
assertThat(
StreamSupport.stream(counterMeter.measure().spliterator(), false).mapToDouble(m -> m.getValue()).sum())
.isEqualTo(13.0);
assertThat(counterMeter.getId().getTags().size()).isEqualTo(2);
assertThat(counterMeter.getId().getTag("foo")).isEqualTo("bar");
assertThat(counterMeter.getId().getTag("gork")).isEqualTo("bork");
}
}

8
consumer/spring-analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/GaugeWithAmountTest.java
View File

@@ -26,12 +26,8 @@ import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Christian Tzolov
*/
@TestPropertySource(properties = {
"analytics.meter-type=gauge",
"analytics.name=myGauge",
"analytics.tag.expression.foo='bar'",
"analytics.amount-expression=payload.length()"
})
@TestPropertySource(properties = { "analytics.meter-type=gauge", "analytics.name=myGauge",
"analytics.tag.expression.foo='bar'", "analytics.amount-expression=payload.length()" })
class GaugeWithAmountTest extends AnalyticsConsumerParentTest {
@Test

7
consumer/spring-analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/LiteralTagExpressionsTests.java
View File

@@ -29,11 +29,8 @@ import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Christian Tzolov
*/
@TestPropertySource(properties = {
"analytics.name=counter666",
"analytics.tag.expression.foo='bar'",
"analytics.tag.expression.gork='bork'"
})
@TestPropertySource(properties = { "analytics.name=counter666", "analytics.tag.expression.foo='bar'",
"analytics.tag.expression.gork='bork'" })
public class LiteralTagExpressionsTests extends AnalyticsConsumerParentTest {
@Test

14
consumer/spring-analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/NullTagsTests.java
View File

@@ -28,12 +28,9 @@ import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Christian Tzolov
*/
@TestPropertySource(properties = {
"analytics.name=counter666",
"analytics.tag.fixed.foo=",
@TestPropertySource(properties = { "analytics.name=counter666", "analytics.tag.fixed.foo=",
"analytics.tag.expression.tag666=#jsonPath(payload,'$..noField')",
"analytics.tag.expression.test=#jsonPath(payload,'$..test')"
})
"analytics.tag.expression.test=#jsonPath(payload,'$..test')" })
public class NullTagsTests extends AnalyticsConsumerParentTest {
@Test
@@ -46,10 +43,13 @@ public class NullTagsTests extends AnalyticsConsumerParentTest {
Collection<Counter> expressionTagsCounters = meterRegistry.find("counter666").tagKeys("tag666").counters();
assertThat(expressionTagsCounters.size()).isEqualTo(1);
assertThat(meterRegistry.find("counter666").meter().getId().getTag("tag666")).isEqualTo(AnalyticsConsumerConfiguration.UNAVAILABLE_TAG);
assertThat(meterRegistry.find("counter666").meter().getId().getTag("tag666"))
.isEqualTo(AnalyticsConsumerConfiguration.UNAVAILABLE_TAG);
Collection<Counter> testExpTagsCounters = meterRegistry.find("counter666").tagKeys("test").counters();
assertThat(testExpTagsCounters.size()).isEqualTo(1);
assertThat(meterRegistry.find("counter666").meter().getId().getTag("test")).isEqualTo(AnalyticsConsumerConfiguration.UNAVAILABLE_TAG);
assertThat(meterRegistry.find("counter666").meter().getId().getTag("test"))
.isEqualTo(AnalyticsConsumerConfiguration.UNAVAILABLE_TAG);
}
}

24
consumer/spring-analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/StockExchangeAnalyticsTests.java
View File

@@ -33,12 +33,9 @@ import static org.assertj.core.api.Assertions.assertThat;
* @author Christian Tzolov
*/
@TestPropertySource(properties = {
"analytics.meter-type=counter",
"analytics.name=stocks",
@TestPropertySource(properties = { "analytics.meter-type=counter", "analytics.name=stocks",
"analytics.tag.expression.symbol=#jsonPath(payload,'$.data.symbol')",
"analytics.tag.expression.exchange=#jsonPath(payload,'$.data.exchange')"
})
"analytics.tag.expression.exchange=#jsonPath(payload,'$.data.exchange')" })
public class StockExchangeAnalyticsTests extends AnalyticsConsumerParentTest {
@Test
@@ -60,15 +57,18 @@ public class StockExchangeAnalyticsTests extends AnalyticsConsumerParentTest {
assertThat(counters).hasSize(2);
//Iterator<Counter> itr = counters.iterator();
// Iterator<Counter> itr = counters.iterator();
//
//Counter applCounter = itr.next();
//assertThat(applCounter.count()).isEqualTo(3);
//assertThat(applCounter.getId().getTags()).contains(Tag.of("symbol", "AAPL"), Tag.of("exchange", "XNAS"));
// Counter applCounter = itr.next();
// assertThat(applCounter.count()).isEqualTo(3);
// assertThat(applCounter.getId().getTags()).contains(Tag.of("symbol", "AAPL"),
// Tag.of("exchange", "XNAS"));
//
//Counter vmwCounter = itr.next();
//assertThat(vmwCounter.count()).isEqualTo(2);
//assertThat(vmwCounter.getId().getTags()).contains(Tag.of("symbol", "VMW"), Tag.of("exchange", "NYSE"));
// Counter vmwCounter = itr.next();
// assertThat(vmwCounter.count()).isEqualTo(2);
// assertThat(vmwCounter.getId().getTags()).contains(Tag.of("symbol", "VMW"),
// Tag.of("exchange", "NYSE"));
}
}

46
consumer/spring-analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/demo/StockExchangeAnalyticsExample.java
View File

@@ -35,19 +35,17 @@ import org.springframework.messaging.Message;
import org.springframework.messaging.support.MessageBuilder;
/**
* Sample Spring Boot Application that uses the analyticsConsumer to compute running stats from
* stock exchange messages.
* Sample Spring Boot Application that uses the analyticsConsumer to compute running stats
* from stock exchange messages.
*
* Counter configuration:
* <code>
* Counter configuration: <code>
* --analytics.meter-type=counter
* --analytics.name=stocks
* --analytics.tag.expression.symbol=#jsonPath(payload,'$.data.symbol')
* --analytics.tag.expression.exchange=#jsonPath(payload,'$.data.exchange')
* </code>
*
* Gauge configuration:
* <code>
* Gauge configuration: <code>
* --analytics.meter-type=gauge
* --analytics.name=stocks
* --analytics.tag.expression.symbol=#jsonPath(payload,'$.data.symbol')
@@ -55,8 +53,7 @@ import org.springframework.messaging.support.MessageBuilder;
* --analytics.amount-expression=#jsonPath(payload,'$.data.volume')
* </code>
*
* Sample Wavefront configuration:
* <code>
* Sample Wavefront configuration: <code>
* --management.metrics.export.wavefront.enabled=true
* --management.metrics.export.wavefront.uri=YOUR_WAVEFRONT_SERVER_URI
* --management.metrics.export.wavefront.api-token=YOUR_API_TOKEN
@@ -74,23 +71,25 @@ public class StockExchangeAnalyticsExample {
}
@Bean
public CommandLineRunner commandLineRunner(Consumer<Message<?>> analyticsConsumer,
MeterRegistry meterRegistry, Supplier<String> stockMessageGenerator) {
public CommandLineRunner commandLineRunner(Consumer<Message<?>> analyticsConsumer, MeterRegistry meterRegistry,
Supplier<String> stockMessageGenerator) {
// Run every second.
return args -> Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> {
String message = stockMessageGenerator.get();
// Submit new message using the stockMessageGenerator to generate random stock messages.
// Submit new message using the stockMessageGenerator to generate random stock
// messages.
analyticsConsumer.accept(MessageBuilder.withPayload(message).build());
// Print current stock meters
System.out.println(meterRegistry.getMeters().stream()
.filter(meter -> meter.getId().getName().contains("stocks"))
.map(meter -> meter.getId().getType() + " | " + meter.getId() + " | " + meter.measure())
.collect(Collectors.joining("\n")) +
"\n=========================================================================");
System.out.println(meterRegistry.getMeters()
.stream()
.filter(meter -> meter.getId().getName().contains("stocks"))
.map(meter -> meter.getId().getType() + " | " + meter.getId() + " | " + meter.measure())
.collect(Collectors.joining("\n"))
+ "\n=========================================================================");
}, 0, 1000, TimeUnit.MILLISECONDS);
}
@@ -103,16 +102,11 @@ public class StockExchangeAnalyticsExample {
return () -> {
int stockIndex = random.nextInt(STOCKS.length);
return "{\n" +
" \"data\": {\n" +
" \"symbol\": \"" + STOCKS[stockIndex][1] + "\",\n" +
" \"exchange\": \"" + STOCKS[stockIndex][0] + "\",\n" +
" \"open\": " + (1 + 10 * random.nextDouble()) + ",\n" +
" \"close\": " + (1 + 10 * random.nextDouble()) + ",\n" +
" \"volume\": " + (1000 + 100000 * random.nextDouble()) + "\n" +
" }\n" +
"}";
return "{\n" + " \"data\": {\n" + " \"symbol\": \"" + STOCKS[stockIndex][1] + "\",\n"
+ " \"exchange\": \"" + STOCKS[stockIndex][0] + "\",\n" + " \"open\": "
+ (1 + 10 * random.nextDouble()) + ",\n" + " \"close\": " + (1 + 10 * random.nextDouble())
+ ",\n" + " \"volume\": " + (1000 + 100000 * random.nextDouble()) + "\n" + " }\n" + "}";
};
}
}
}

47
consumer/spring-cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerConfiguration.java
View File

@@ -75,22 +75,17 @@ public class CassandraConsumerConfiguration {
IntegrationFlowBuilder integrationFlowBuilder = IntegrationFlow.from(CassandraConsumerFunction.class);
String ingestQuery = this.cassandraSinkProperties.getIngestQuery();
if (StringUtils.hasText(ingestQuery)) {
integrationFlowBuilder.transform(
new PayloadToMatrixTransformer(objectMapper, ingestQuery,
CassandraMessageHandler.Type.UPDATE == this.cassandraSinkProperties.getQueryType()
? new UpdateQueryColumnNameExtractor()
: new InsertQueryColumnNameExtractor()));
integrationFlowBuilder.transform(new PayloadToMatrixTransformer(objectMapper, ingestQuery,
CassandraMessageHandler.Type.UPDATE == this.cassandraSinkProperties.getQueryType()
? new UpdateQueryColumnNameExtractor() : new InsertQueryColumnNameExtractor()));
}
return integrationFlowBuilder
.handle(cassandraSinkMessageHandler)
.get();
return integrationFlowBuilder.handle(cassandraSinkMessageHandler).get();
}
@Bean
public MessageHandler cassandraSinkMessageHandler(ReactiveCassandraOperations cassandraOperations) {
CassandraMessageHandler.Type queryType =
Optional.ofNullable(this.cassandraSinkProperties.getQueryType())
.orElse(CassandraMessageHandler.Type.INSERT);
CassandraMessageHandler.Type queryType = Optional.ofNullable(this.cassandraSinkProperties.getQueryType())
.orElse(CassandraMessageHandler.Type.INSERT);
CassandraMessageHandler cassandraMessageHandler = new CassandraMessageHandler(cassandraOperations, queryType);
cassandraMessageHandler.setProducesReply(true);
@@ -98,24 +93,22 @@ public class CassandraConsumerConfiguration {
ConsistencyLevel consistencyLevel = this.cassandraSinkProperties.getConsistencyLevel();
if (consistencyLevel != null || ttl > 0) {
WriteOptions.WriteOptionsBuilder writeOptionsBuilder =
switch (queryType) {
case INSERT -> InsertOptions.builder();
case UPDATE -> UpdateOptions.builder();
default -> WriteOptions.builder();
};
WriteOptions.WriteOptionsBuilder writeOptionsBuilder = switch (queryType) {
case INSERT -> InsertOptions.builder();
case UPDATE -> UpdateOptions.builder();
default -> WriteOptions.builder();
};
JavaUtils.INSTANCE
.acceptIfNotNull(consistencyLevel, writeOptionsBuilder::consistencyLevel)
.acceptIfCondition(ttl > 0, ttl, writeOptionsBuilder::ttl);
JavaUtils.INSTANCE.acceptIfNotNull(consistencyLevel, writeOptionsBuilder::consistencyLevel)
.acceptIfCondition(ttl > 0, ttl, writeOptionsBuilder::ttl);
cassandraMessageHandler.setWriteOptions(writeOptionsBuilder.build());
}
JavaUtils.INSTANCE
.acceptIfHasText(this.cassandraSinkProperties.getIngestQuery(), cassandraMessageHandler::setIngestQuery)
.acceptIfNotNull(this.cassandraSinkProperties.getStatementExpression(),
cassandraMessageHandler::setStatementExpression);
.acceptIfHasText(this.cassandraSinkProperties.getIngestQuery(), cassandraMessageHandler::setIngestQuery)
.acceptIfNotNull(this.cassandraSinkProperties.getStatementExpression(),
cassandraMessageHandler::setStatementExpression);
return cassandraMessageHandler;
}
@@ -124,15 +117,13 @@ public class CassandraConsumerConfiguration {
if (uuid.length() == 36) {
String[] parts = uuid.split("-");
if (parts.length == 5) {
return (parts[0].length() == 8) && (parts[1].length() == 4) &&
(parts[2].length() == 4) && (parts[3].length() == 4) &&
(parts[4].length() == 12);
return (parts[0].length() == 8) && (parts[1].length() == 4) && (parts[2].length() == 4)
&& (parts[3].length() == 4) && (parts[4].length() == 12);
}
}
return false;
}
private static class PayloadToMatrixTransformer extends AbstractPayloadTransformer<Object, List<List<Object>>> {
private final Jackson2JsonObjectMapper jsonObjectMapper;
@@ -145,7 +136,7 @@ public class CassandraConsumerConfiguration {
this.jsonObjectMapper = new Jackson2JsonObjectMapper(objectMapper);
this.columns.addAll(columnNameExtractor.extract(query));
this.jsonObjectMapper.getObjectMapper()
.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
}
@Override

60
consumer/spring-cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraAppClusterConfiguration.java
View File

@@ -63,36 +63,31 @@ import org.springframework.util.StringUtils;
public class CassandraAppClusterConfiguration {
@Bean
public CqlSessionBuilderCustomizer clusterBuilderCustomizer(
CassandraClusterProperties cassandraClusterProperties) {
public CqlSessionBuilderCustomizer clusterBuilderCustomizer(CassandraClusterProperties cassandraClusterProperties) {
PropertyMapper map = PropertyMapper.get();
return builder ->
map.from(cassandraClusterProperties::isSkipSslValidation)
.whenTrue()
.toCall(() -> {
try {
builder.withSslContext(TrustAllSSLContextFactory.getSslContext());
}
catch (NoSuchAlgorithmException | KeyManagementException ex) {
throw new BeanInitializationException(
"Unable to configure a Cassandra cluster using SSL.", ex);
}
return builder -> map.from(cassandraClusterProperties::isSkipSslValidation).whenTrue().toCall(() -> {
try {
builder.withSslContext(TrustAllSSLContextFactory.getSslContext());
}
catch (NoSuchAlgorithmException | KeyManagementException ex) {
throw new BeanInitializationException("Unable to configure a Cassandra cluster using SSL.", ex);
}
});
});
}
@Bean
@ConditionalOnProperty("cassandra.cluster.create-keyspace")
public Object keyspaceCreator(CassandraProperties cassandraProperties, CqlSessionBuilder cqlSessionBuilder) {
CreateKeyspaceSpecification createKeyspaceSpecification =
CreateKeyspaceSpecification
.createKeyspace(cassandraProperties.getKeyspaceName())
.withSimpleReplication()
.ifNotExists();
CreateKeyspaceSpecification createKeyspaceSpecification = CreateKeyspaceSpecification
.createKeyspace(cassandraProperties.getKeyspaceName())
.withSimpleReplication()
.ifNotExists();
String createKeySpaceQuery = new CreateKeyspaceCqlGenerator(createKeyspaceSpecification).toCql();
try (var systemSession = cqlSessionBuilder.withKeyspace(CqlSessionFactoryBean.CASSANDRA_SYSTEM_SESSION).build()) {
try (var systemSession = cqlSessionBuilder.withKeyspace(CqlSessionFactoryBean.CASSANDRA_SYSTEM_SESSION)
.build()) {
systemSession.execute(createKeySpaceQuery);
}
@@ -106,25 +101,22 @@ public class CassandraAppClusterConfiguration {
return cqlSessionBuilder.build();
}
@Bean
@ConditionalOnProperty("cassandra.cluster.init-script")
public Object keyspaceInitializer(CassandraClusterProperties cassandraClusterProperties,
ReactiveCassandraTemplate reactiveCassandraTemplate) throws IOException {
String scripts =
new Scanner(cassandraClusterProperties.getInitScript().getInputStream(),
StandardCharsets.UTF_8)
.useDelimiter("\\A")
.next();
String scripts = new Scanner(cassandraClusterProperties.getInitScript().getInputStream(),
StandardCharsets.UTF_8)
.useDelimiter("\\A")
.next();
ReactiveCqlOperations reactiveCqlOperations =
reactiveCassandraTemplate.getReactiveCqlOperations();
ReactiveCqlOperations reactiveCqlOperations = reactiveCassandraTemplate.getReactiveCqlOperations();
Flux.fromArray(StringUtils.delimitedListToStringArray(scripts, ";", "\r\n\f"))
.filter(StringUtils::hasText) // an empty String after the last ';'
.concatMap(script -> reactiveCqlOperations.execute(script + ";"))
.blockLast();
.filter(StringUtils::hasText) // an empty String after the last ';'
.concatMap(script -> reactiveCqlOperations.execute(script + ";"))
.blockLast();
return null;
@@ -144,9 +136,9 @@ public class CassandraAppClusterConfiguration {
BeanDefinitionRegistry registry) {
Binder.get(this.environment)
.bind("cassandra.cluster.entity-base-packages", String[].class)
.map(Arrays::asList)
.ifBound(packagesToScan -> EntityScanPackages.register(registry, packagesToScan));
.bind("cassandra.cluster.entity-base-packages", String[].class)
.map(Arrays::asList)
.ifBound(packagesToScan -> EntityScanPackages.register(registry, packagesToScan));
}
}

3
consumer/spring-cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraClusterProperties.java
View File

@@ -47,8 +47,7 @@ public class CassandraClusterProperties {
/**
* Base packages to scan for entities annotated with Table annotations.
*/
private String[] entityBasePackages = { };
private String[] entityBasePackages = {};
public void setCreateKeyspace(boolean createKeyspace) {
this.createKeyspace = createKeyspace;

31
consumer/spring-cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/TrustAllSSLContextFactory.java
View File

@@ -25,10 +25,9 @@ import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
/**
* Helper to provide an SSL Context that does not validate
* certificates presented in the SSL handshake.
* Helper to provide an SSL Context that does not validate certificates presented in the
* SSL handshake.
*
* The usual caveats apply.
*
@@ -43,24 +42,22 @@ final class TrustAllSSLContextFactory {
static SSLContext getSslContext() throws NoSuchAlgorithmException, KeyManagementException {
TrustManager[] trustAllCerts = new TrustManager[] {
new X509TrustManager() {
TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() {
@Override
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
@Override
public void checkClientTrusted(X509Certificate[] certs, String authType) {
}
@Override
public void checkClientTrusted(X509Certificate[] certs, String authType) {
}
@Override
public void checkServerTrusted(X509Certificate[] certs, String authType) {
}
@Override
public void checkServerTrusted(X509Certificate[] certs, String authType) {
}
}
};
} };
SSLContext sc = SSLContext.getInstance("SSL");
sc.init(null, trustAllCerts, new SecureRandom());

18
consumer/spring-cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerApplicationTests.java
View File

@@ -42,8 +42,7 @@ import org.springframework.test.context.DynamicPropertySource;
* @author Artem Bilan
*/
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE,
properties = {
"spring.cassandra.keyspace-name=" + CassandraConsumerApplicationTests.CASSANDRA_KEYSPACE,
properties = { "spring.cassandra.keyspace-name=" + CassandraConsumerApplicationTests.CASSANDRA_KEYSPACE,
"cassandra.cluster.createKeyspace=true" })
@DirtiesContext
abstract class CassandraConsumerApplicationTests implements CassandraContainerTest {
@@ -56,12 +55,11 @@ abstract class CassandraConsumerApplicationTests implements CassandraContainerTe
@Autowired
protected Function<Object, Mono<? extends WriteResult>> cassandraConsumer;
@DynamicPropertySource
static void registerConfigurationProperties(DynamicPropertyRegistry registry) {
registry.add("spring.cassandra.localDatacenter", () -> CASSANDRA_CONTAINER.getLocalDatacenter());
registry.add("spring.cassandra.contactPoints", () ->
Optional.of(CASSANDRA_CONTAINER.getContactPoint())
registry.add("spring.cassandra.contactPoints",
() -> Optional.of(CASSANDRA_CONTAINER.getContactPoint())
.map(contactPoint -> contactPoint.getAddress().getHostAddress() + ':' + contactPoint.getPort())
.get());
}
@@ -75,14 +73,8 @@ abstract class CassandraConsumerApplicationTests implements CassandraContainerTe
List<Book> books = new ArrayList<>();
for (int i = 0; i < numBooks; i++) {
books.add(
new Book(
UUID.randomUUID(),
"Spring Cloud Data Flow Guide",
"SCDF Guru",
i * 10 + 5,
LocalDate.now(),
true));
books.add(new Book(UUID.randomUUID(), "Spring Cloud Data Flow Guide", "SCDF Guru", i * 10 + 5,
LocalDate.now(), true));
}
return books;

11
consumer/spring-cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraContainerTest.java
View File

@@ -21,14 +21,13 @@ import org.testcontainers.containers.CassandraContainer;
import org.testcontainers.junit.jupiter.Testcontainers;
/**
* The base contract for JUnit tests based on the container for Apache Cassandra.
* The Testcontainers 'reuse' option must be disabled,so, Ryuk container is started
* and will clean all the containers up from this test suite after JVM exit.
* Since the MqSQL container instance is shared via static property, it is going to be
* started only once per JVM, therefore the target Docker container is reused automatically.
* The base contract for JUnit tests based on the container for Apache Cassandra. The
* Testcontainers 'reuse' option must be disabled,so, Ryuk container is started and will
* clean all the containers up from this test suite after JVM exit. Since the MqSQL
* container instance is shared via static property, it is going to be started only once
* per JVM, therefore the target Docker container is reused automatically.
*
* @author Artem Bilan
*
* @since 6.0
*/
@Testcontainers(disabledWithoutDocker = true)

22
consumer/spring-cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraEntityInsertTests.java
View File

@@ -32,31 +32,21 @@ import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Artem Bilan
*/
@TestPropertySource(properties = {
"spring.cassandra.schema-action=RECREATE",
@TestPropertySource(properties = { "spring.cassandra.schema-action=RECREATE",
"cassandra.cluster.entity-base-packages=org.springframework.cloud.fn.consumer.cassandra.domain" })
class CassandraEntityInsertTests extends CassandraConsumerApplicationTests {
@Test
void testInsert() {
Book book =
new Book(
UUID.randomUUID(),
"Spring Integration Cassandra",
"Cassandra Guru",
521,
LocalDate.now(),
true);
Book book = new Book(UUID.randomUUID(), "Spring Integration Cassandra", "Cassandra Guru", 521, LocalDate.now(),
true);
Mono<? extends WriteResult> result = this.cassandraConsumer.apply(book);
StepVerifier.create(result)
.expectNextCount(1)
.then(() ->
assertThat(this.cassandraTemplate.query(Book.class)
.count())
.isEqualTo(1))
.verifyComplete();
.expectNextCount(1)
.then(() -> assertThat(this.cassandraTemplate.query(Book.class).count()).isEqualTo(1))
.verifyComplete();
}
}

18
consumer/spring-cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestInsertTests.java
View File

@@ -34,10 +34,8 @@ import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Artem Bilan
*/
@TestPropertySource(properties = {
"cassandra.cluster.init-script=init-db.cql",
"cassandra.ingest-query=" +
"insert into book (isbn, title, author, pages, saleDate, inStock) values (?, ?, ?, ?, ?, ?)" })
@TestPropertySource(properties = { "cassandra.cluster.init-script=init-db.cql", "cassandra.ingest-query="
+ "insert into book (isbn, title, author, pages, saleDate, inStock) values (?, ?, ?, ?, ?, ?)" })
class CassandraIngestInsertTests extends CassandraConsumerApplicationTests {
@Test
@@ -46,16 +44,12 @@ class CassandraIngestInsertTests extends CassandraConsumerApplicationTests {
Jackson2JsonObjectMapper mapper = new Jackson2JsonObjectMapper(objectMapper);
Mono<? extends WriteResult> result =
this.cassandraConsumer.apply(mapper.toJson(books));
Mono<? extends WriteResult> result = this.cassandraConsumer.apply(mapper.toJson(books));
StepVerifier.create(result)
.expectNextCount(1)
.then(() ->
assertThat(this.cassandraTemplate.query(Book.class)
.count())
.isEqualTo(5))
.verifyComplete();
.expectNextCount(1)
.then(() -> assertThat(this.cassandraTemplate.query(Book.class).count()).isEqualTo(5))
.verifyComplete();
}
}

20
consumer/spring-cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestNamedParamsTests.java
View File

@@ -35,11 +35,9 @@ import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Artem Bilan
*/
@TestPropertySource(properties = {
"cassandra.cluster.init-script=init-db.cql",
"cassandra.ingest-query=" +
"insert into book (isbn, title, author, pages, saleDate, inStock) " +
"values (:myIsbn, :myTitle, :myAuthor, ?, ?, ?)" })
@TestPropertySource(properties = { "cassandra.cluster.init-script=init-db.cql",
"cassandra.ingest-query=" + "insert into book (isbn, title, author, pages, saleDate, inStock) "
+ "values (:myIsbn, :myTitle, :myAuthor, ?, ?, ?)" })
class CassandraIngestNamedParamsTests extends CassandraConsumerApplicationTests {
@Test
@@ -53,16 +51,12 @@ class CassandraIngestNamedParamsTests extends CassandraConsumerApplicationTests
booksJsonWithNamedParams = StringUtils.replace(booksJsonWithNamedParams, "title", "myTitle");
booksJsonWithNamedParams = StringUtils.replace(booksJsonWithNamedParams, "author", "myAuthor");
Mono<? extends WriteResult> result =
this.cassandraConsumer.apply(booksJsonWithNamedParams);
Mono<? extends WriteResult> result = this.cassandraConsumer.apply(booksJsonWithNamedParams);
StepVerifier.create(result)
.expectNextCount(1)
.then(() ->
assertThat(this.cassandraTemplate.query(Book.class)
.count())
.isEqualTo(5))
.verifyComplete();
.expectNextCount(1)
.then(() -> assertThat(this.cassandraTemplate.query(Book.class).count()).isEqualTo(5))
.verifyComplete();
}
}

20
consumer/spring-cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestUpdateTests.java
View File

@@ -34,11 +34,9 @@ import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Artem Bilan
*/
@TestPropertySource(properties = {
"cassandra.cluster.init-script=init-db.cql",
"cassandra.ingest-query=" +
"update book set inStock = :inStock, author = :author, pages = :pages, " +
"saleDate = :saleDate, title = :title where isbn = :isbn",
@TestPropertySource(properties = { "cassandra.cluster.init-script=init-db.cql",
"cassandra.ingest-query=" + "update book set inStock = :inStock, author = :author, pages = :pages, "
+ "saleDate = :saleDate, title = :title where isbn = :isbn",
"cassandra.queryType=UPDATE" })
class CassandraIngestUpdateTests extends CassandraConsumerApplicationTests {
@@ -48,16 +46,12 @@ class CassandraIngestUpdateTests extends CassandraConsumerApplicationTests {
Jackson2JsonObjectMapper mapper = new Jackson2JsonObjectMapper(objectMapper);
Mono<? extends WriteResult> result =
this.cassandraConsumer.apply(mapper.toJson(books));
Mono<? extends WriteResult> result = this.cassandraConsumer.apply(mapper.toJson(books));
StepVerifier.create(result)
.expectNextCount(1)
.then(() ->
assertThat(this.cassandraTemplate.query(Book.class)
.count())
.isEqualTo(5))
.verifyComplete();
.expectNextCount(1)
.then(() -> assertThat(this.cassandraTemplate.query(Book.class).count()).isEqualTo(5))
.verifyComplete();
}
}

7
consumer/spring-cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/domain/Book.java
View File

@@ -30,12 +30,7 @@ import org.springframework.data.cassandra.core.mapping.Table;
* @author Artem Bilan
*/
@Table("book")
public record Book(
@PrimaryKey UUID isbn,
String title,
@Indexed String author,
Integer pages,
LocalDate saleDate,
public record Book(@PrimaryKey UUID isbn, String title, @Indexed String author, Integer pages, LocalDate saleDate,
Boolean isInStock) {
}

79
consumer/spring-elasticsearch-consumer/src/main/java/org/springframework/cloud/fn/consumer/elasticsearch/ElasticsearchConsumerConfiguration.java
View File

@@ -78,19 +78,24 @@ public class ElasticsearchConsumerConfiguration {
private static final Log logger = LogFactory.getLog(ElasticsearchConsumerConfiguration.class);
@Bean
FactoryBean<MessageHandler> aggregator(MessageGroupStore messageGroupStore, ElasticsearchConsumerProperties consumerProperties) {
FactoryBean<MessageHandler> aggregator(MessageGroupStore messageGroupStore,
ElasticsearchConsumerProperties consumerProperties) {
AggregatorFactoryBean aggregatorFactoryBean = new AggregatorFactoryBean();
aggregatorFactoryBean.setCorrelationStrategy(message -> "");
aggregatorFactoryBean.setReleaseStrategy(new MessageCountReleaseStrategy(consumerProperties.getBatchSize()));
if (consumerProperties.getGroupTimeout() >= 0) {
aggregatorFactoryBean.setGroupTimeoutExpression(new ValueExpression<>(consumerProperties.getGroupTimeout()));
aggregatorFactoryBean
.setGroupTimeoutExpression(new ValueExpression<>(consumerProperties.getGroupTimeout()));
}
aggregatorFactoryBean.setMessageStore(messageGroupStore);
// Currently, there is no way to customize the splitting behavior of an aggregator receiving
// Currently, there is no way to customize the splitting behavior of an aggregator
// receiving
// a Collection<Message<?>> from the configured MessageGroupProcessor.
// Thus, fooling the aggregator with a wrapper of Message<?> is just a straightforward way to preserve the
// individual message headers and release an entire batch to downstream indexing handler.
// Thus, fooling the aggregator with a wrapper of Message<?> is just a
// straightforward way to preserve the
// individual message headers and release an entire batch to downstream indexing
// handler.
aggregatorFactoryBean.setProcessorBean(new AbstractAggregatingMessageGroupProcessor() {
@Override
protected Object aggregatePayloads(MessageGroup group, Map<String, Object> defaultHeaders) {
@@ -118,14 +123,11 @@ public class ElasticsearchConsumerConfiguration {
}
@Bean
IntegrationFlow elasticsearchConsumerFlow(
@Qualifier("aggregator") MessageHandler aggregator,
ElasticsearchConsumerProperties properties,
@Qualifier("indexingHandler") MessageHandler indexingHandler
) {
IntegrationFlow elasticsearchConsumerFlow(@Qualifier("aggregator") MessageHandler aggregator,
ElasticsearchConsumerProperties properties, @Qualifier("indexingHandler") MessageHandler indexingHandler) {
final IntegrationFlowBuilder builder =
IntegrationFlow.from(MessageConsumer.class, gateway -> gateway.beanName("elasticsearchConsumer"));
final IntegrationFlowBuilder builder = IntegrationFlow.from(MessageConsumer.class,
gateway -> gateway.beanName("elasticsearchConsumer"));
if (properties.getBatchSize() > 1) {
builder.handle(aggregator);
}
@@ -133,10 +135,8 @@ public class ElasticsearchConsumerConfiguration {
}
@Bean
public MessageHandler indexingHandler(
ElasticsearchClient elasticsearchClient,
ElasticsearchConsumerProperties consumerProperties
) {
public MessageHandler indexingHandler(ElasticsearchClient elasticsearchClient,
ElasticsearchConsumerProperties consumerProperties) {
return message -> {
if (message.getPayload() instanceof Iterable) {
BulkRequest.Builder builder = new BulkRequest.Builder();
@@ -144,9 +144,10 @@ public class ElasticsearchConsumerConfiguration {
.filter(MessageWrapper.class::isInstance)
.map(itemPayload -> ((MessageWrapper) itemPayload).getMessage())
.map(m -> buildIndexRequest(m, consumerProperties))
.forEach(indexRequest ->
builder.operations(builder1 -> builder1.index(idx -> idx.index(indexRequest.index()).id(indexRequest.id()).document(indexRequest.document())))
);
.forEach(indexRequest -> builder
.operations(builder1 -> builder1.index(idx -> idx.index(indexRequest.index())
.id(indexRequest.id())
.document(indexRequest.document()))));
index(elasticsearchClient, builder.build(), consumerProperties.isAsync());
}
@@ -194,11 +195,13 @@ public class ElasticsearchConsumerConfiguration {
private void index(ElasticsearchClient elasticsearchClient, BulkRequest request, boolean isAsync) {
if (isAsync) {
ElasticsearchAsyncClient elasticsearchAsyncClient = new ElasticsearchAsyncClient(elasticsearchClient._transport());
ElasticsearchAsyncClient elasticsearchAsyncClient = new ElasticsearchAsyncClient(
elasticsearchClient._transport());
CompletableFuture<BulkResponse> responseCompletableFuture = elasticsearchAsyncClient.bulk(request);
responseCompletableFuture.whenComplete((bulkResponse, x) -> {
if (x != null) {
throw new IllegalStateException("Error occurred while performing bulk index operation: " + x.getMessage(), x);
throw new IllegalStateException(
"Error occurred while performing bulk index operation: " + x.getMessage(), x);
}
else {
handleBulkResponse(bulkResponse);
@@ -212,14 +215,16 @@ public class ElasticsearchConsumerConfiguration {
handleBulkResponse(bulkResponse);
}
catch (IOException e) {
throw new IllegalStateException("Error occurred while performing bulk index operation: " + e.getMessage(), e);
throw new IllegalStateException(
"Error occurred while performing bulk index operation: " + e.getMessage(), e);
}
}
}
private void index(ElasticsearchClient elasticsearchClient, IndexRequest request, boolean isAsync) {
if (isAsync) {
ElasticsearchAsyncClient elasticsearchAsyncClient = new ElasticsearchAsyncClient(elasticsearchClient._transport());
ElasticsearchAsyncClient elasticsearchAsyncClient = new ElasticsearchAsyncClient(
elasticsearchClient._transport());
CompletableFuture<IndexResponse> responseCompletableFuture = elasticsearchAsyncClient.index(request);
responseCompletableFuture.whenComplete((indexResponse, x) -> {
if (x != null) {
@@ -248,9 +253,8 @@ public class ElasticsearchConsumerConfiguration {
if (logger.isDebugEnabled()) {
logger.debug("itemResponse.error=" + itemResponse.error());
}
logger.error(String.format("Index operation [id=%s, index=%s] failed: %s",
itemResponse.id(), itemResponse.index(), itemResponse.error().toString())
);
logger.error(String.format("Index operation [id=%s, index=%s] failed: %s", itemResponse.id(),
itemResponse.index(), itemResponse.error().toString()));
}
else {
var r = itemResponse.get();
@@ -258,12 +262,14 @@ public class ElasticsearchConsumerConfiguration {
if (logger.isDebugEnabled()) {
logger.debug("itemResponse:" + r);
}
logger.debug(String.format("Index operation [id=%s, index=%s] succeeded: document [id=%s, version=%s] was written on shard %s.",
itemResponse.id(), itemResponse.index(), r.source().get("id"), r.source().get("version"), r.source().get("shardId"))
);
logger.debug(String.format(
"Index operation [id=%s, index=%s] succeeded: document [id=%s, version=%s] was written on shard %s.",
itemResponse.id(), itemResponse.index(), r.source().get("id"),
r.source().get("version"), r.source().get("shardId")));
}
else {
logger.debug(String.format("Index operation [id=%s, index=%s] succeeded", itemResponse.id(), itemResponse.index()));
logger.debug(String.format("Index operation [id=%s, index=%s] succeeded", itemResponse.id(),
itemResponse.index()));
}
}
}
@@ -272,20 +278,22 @@ public class ElasticsearchConsumerConfiguration {
if (response.errors()) {
String error = response.items()
.stream()
.map(bulkResponseItem -> bulkResponseItem.error() != null ? bulkResponseItem.error().toString() : "")
.reduce((errorCause, errorCause2) -> errorCause != null ? errorCause + " : " + errorCause2 : errorCause2)
.map(bulkResponseItem -> bulkResponseItem.error() != null ? bulkResponseItem.error().toString() : "")
.reduce((errorCause, errorCause2) -> errorCause != null ? errorCause + " : " + errorCause2
: errorCause2)
.orElseGet(response::toString);
throw new IllegalStateException("Bulk indexing operation completed with failures: " + error);
}
}
private void handleResponse(IndexResponse response) {
logger.debug(String.format("Index operation [index=%s] succeeded: document [id=%s, version=%d] was written on shard %s.",
response.index(), response.id(), response.version(), response.shards().toString())
);
logger.debug(String.format(
"Index operation [index=%s] succeeded: document [id=%s, version=%d] was written on shard %s.",
response.index(), response.id(), response.version(), response.shards().toString()));
}
static class MessageWrapper {
private final Message<?> message;
MessageWrapper(Message<?> message) {
@@ -295,6 +303,7 @@ public class ElasticsearchConsumerConfiguration {
public Message<?> getMessage() {
return message;
}
}
private interface MessageConsumer extends Consumer<Message<?>> {

30
consumer/spring-elasticsearch-consumer/src/main/java/org/springframework/cloud/fn/consumer/elasticsearch/ElasticsearchConsumerProperties.java
View File

@@ -27,44 +27,45 @@ import org.springframework.expression.Expression;
public class ElasticsearchConsumerProperties {
/**
* The id of the document to index.
* If set, the INDEX_ID header value overrides this property on a per message basis.
* The id of the document to index. If set, the INDEX_ID header value overrides this
* property on a per message basis.
*/
Expression id;
/**
* Name of the index.
* If set, the INDEX_NAME header value overrides this property on a per message basis.
* Name of the index. If set, the INDEX_NAME header value overrides this property on a
* per message basis.
*/
String index;
/**
* Indicates the shard to route to.
* If not provided, Elasticsearch will default to a hash of the document id.
* Indicates the shard to route to. If not provided, Elasticsearch will default to a
* hash of the document id.
*/
String routing;
/**
* Timeout for the shard to be available.
* If not set, it defaults to 1 minute set by the Elasticsearch client.
* Timeout for the shard to be available. If not set, it defaults to 1 minute set by
* the Elasticsearch client.
*/
long timeoutSeconds;
/**
* Indicates whether the indexing operation is async or not.
* By default indexing is done synchronously.
* Indicates whether the indexing operation is async or not. By default indexing is
* done synchronously.
*/
boolean async;
/**
* Number of items to index for each request. It defaults to 1.
* For values greater than 1 bulk indexing API will be used.
* Number of items to index for each request. It defaults to 1. For values greater
* than 1 bulk indexing API will be used.
*/
int batchSize = 1;
/**
* Timeout in milliseconds after which message group is flushed when bulk indexing is active.
* It defaults to -1, meaning no automatic flush of idle message groups occurs.
* Timeout in milliseconds after which message group is flushed when bulk indexing is
* active. It defaults to -1, meaning no automatic flush of idle message groups
* occurs.
*/
long groupTimeout = -1L;
@@ -123,4 +124,5 @@ public class ElasticsearchConsumerProperties {
public void setGroupTimeout(long groupTimeout) {
this.groupTimeout = groupTimeout;
}
}

113
consumer/spring-elasticsearch-consumer/src/test/java/org/springframework/cloud/fn/consumer/elasticsearch/ElasticsearchConsumerApplicationTests.java
View File

@@ -65,9 +65,8 @@ public class ElasticsearchConsumerApplicationTests {
@Container
static final ElasticsearchContainer elasticsearch = new ElasticsearchContainer(
DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch")
.withTag("7.17.7")
).withStartupTimeout(Duration.ofSeconds(120))
DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch").withTag("7.17.7"))
.withStartupTimeout(Duration.ofSeconds(120))
.withStartupAttempts(3);
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
@@ -78,12 +77,12 @@ public class ElasticsearchConsumerApplicationTests {
public void testBasicJsonString() {
this.contextRunner
.withPropertyValues("elasticsearch.consumer.index=foo", "elasticsearch.consumer.id=1",
"spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress())
"spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress())
.run(context -> {
final Consumer<Message<?>> elasticsearchConsumer = context.getBean("elasticsearchConsumer", Consumer.class);
final Consumer<Message<?>> elasticsearchConsumer = context.getBean("elasticsearchConsumer",
Consumer.class);
final String jsonObject = "{\"age\":10,\"dateOfBirth\":1471466076564,"
+ "\"fullName\":\"John Doe\"}";
final String jsonObject = "{\"age\":10,\"dateOfBirth\":1471466076564," + "\"fullName\":\"John Doe\"}";
final Message<String> message = MessageBuilder.withPayload(jsonObject).build();
elasticsearchConsumer.accept(message);
@@ -102,14 +101,15 @@ public class ElasticsearchConsumerApplicationTests {
public void testIdPassedAsMessageHeader() {
this.contextRunner
.withPropertyValues("elasticsearch.consumer.index=foo",
"spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress())
"spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress())
.run(context -> {
final Consumer<Message<?>> elasticsearchConsumer = context.getBean("elasticsearchConsumer", Consumer.class);
final Consumer<Message<?>> elasticsearchConsumer = context.getBean("elasticsearchConsumer",
Consumer.class);
final String jsonObject = "{\"age\":10,\"dateOfBirth\":1471466076564,"
+ "\"fullName\":\"John Doe\"}";
final String jsonObject = "{\"age\":10,\"dateOfBirth\":1471466076564," + "\"fullName\":\"John Doe\"}";
final Message<String> message = MessageBuilder.withPayload(jsonObject)
.setHeader(ElasticsearchConsumerConfiguration.INDEX_ID_HEADER, "2").build();
.setHeader(ElasticsearchConsumerConfiguration.INDEX_ID_HEADER, "2")
.build();
log.info("elasticsearchConsumer.accept:{}", message);
elasticsearchConsumer.accept(message);
@@ -124,13 +124,14 @@ public class ElasticsearchConsumerApplicationTests {
}
@Test
@SuppressWarnings({ "unchecked", "rawtypes"})
@SuppressWarnings({ "unchecked", "rawtypes" })
public void testJsonAsMap() {
this.contextRunner
.withPropertyValues("elasticsearch.consumer.index=foo", "elasticsearch.consumer.id=3",
"spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress())
"spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress())
.run(context -> {
final Consumer<Message<?>> elasticsearchConsumer = context.getBean("elasticsearchConsumer", Consumer.class);
final Consumer<Message<?>> elasticsearchConsumer = context.getBean("elasticsearchConsumer",
Consumer.class);
final Map<String, Object> jsonMap = new HashMap<>();
jsonMap.put("age", 10);
@@ -160,13 +161,13 @@ public class ElasticsearchConsumerApplicationTests {
public void testAsyncIndexing() {
this.contextRunner
.withPropertyValues("elasticsearch.consumer.index=foo", "elasticsearch.consumer.async=true",
"elasticsearch.consumer.id=5",
"spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress())
"elasticsearch.consumer.id=5",
"spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress())
.run(context -> {
final Consumer<Message<?>> elasticsearchConsumer = context.getBean("elasticsearchConsumer", Consumer.class);
final Consumer<Message<?>> elasticsearchConsumer = context.getBean("elasticsearchConsumer",
Consumer.class);
final String jsonObject = "{\"age\":10,\"dateOfBirth\":1471466076564,"
+ "\"fullName\":\"John Doe\"}";
final String jsonObject = "{\"age\":10,\"dateOfBirth\":1471466076564," + "\"fullName\":\"John Doe\"}";
final Message<String> message = MessageBuilder.withPayload(jsonObject).build();
log.info("elasticsearchConsumer.accept:{}", message);
elasticsearchConsumer.accept(message);
@@ -185,16 +186,20 @@ public class ElasticsearchConsumerApplicationTests {
@SuppressWarnings("unchecked")
public void testBulkIndexingWithIdFromHeader() {
this.contextRunner
.withPropertyValues("elasticsearch.consumer.index=foo_" + UUID.randomUUID(), "elasticsearch.consumer.batch-size=10",
"spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress())
.withPropertyValues("elasticsearch.consumer.index=foo_" + UUID.randomUUID(),
"elasticsearch.consumer.batch-size=10",
"spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress())
.run(context -> {
final Consumer<Message<?>> elasticsearchConsumer = context.getBean("elasticsearchConsumer", Consumer.class);
final ElasticsearchConsumerProperties properties = context.getBean(ElasticsearchConsumerProperties.class);
final Consumer<Message<?>> elasticsearchConsumer = context.getBean("elasticsearchConsumer",
Consumer.class);
final ElasticsearchConsumerProperties properties = context
.getBean(ElasticsearchConsumerProperties.class);
final ElasticsearchClient elasticsearchClient = context.getBean(ElasticsearchClient.class);
for (int i = 0; i < properties.getBatchSize(); i++) {
final GetRequest getRequest = new GetRequest.Builder().index(properties.getIndex()).id(Integer.toString(i)).build();
final GetRequest getRequest = new GetRequest.Builder().index(properties.getIndex())
.id(Integer.toString(i))
.build();
assertThatExceptionOfType(ElasticsearchException.class)
.isThrownBy(() -> elasticsearchClient.get(getRequest, JsonData.class))
.withFailMessage("Expected index not found exception for message %d")
@@ -202,7 +207,7 @@ public class ElasticsearchConsumerApplicationTests {
final Message<String> message = MessageBuilder
.withPayload("{\"seq\":" + i + ",\"age\":10,\"dateOfBirth\":1471466076564,"
+ "\"fullName\":\"John Doe\"}")
+ "\"fullName\":\"John Doe\"}")
.setHeader(ElasticsearchConsumerConfiguration.INDEX_ID_HEADER, Integer.toString(i))
.build();
log.info("elasticsearchConsumer.accept:{}", message);
@@ -210,13 +215,14 @@ public class ElasticsearchConsumerApplicationTests {
}
for (int i = 0; i < properties.getBatchSize(); i++) {
final GetRequest getRequest = new GetRequest.Builder().index(properties.getIndex()).id(Integer.toString(i)).build();
final GetRequest getRequest = new GetRequest.Builder().index(properties.getIndex())
.id(Integer.toString(i))
.build();
final GetResponse<JsonData> response = elasticsearchClient.get(getRequest, JsonData.class);
assertThat(response.found())
.withFailMessage("Document with id=%d cannot be found.", i)
.isTrue();
assertThat(response.source().toJson().asJsonObject().get("seq").toString()).isEqualTo(Integer.toString(i));
assertThat(response.found()).withFailMessage("Document with id=%d cannot be found.", i).isTrue();
assertThat(response.source().toJson().asJsonObject().get("seq").toString())
.isEqualTo(Integer.toString(i));
}
});
}
@@ -225,16 +231,20 @@ public class ElasticsearchConsumerApplicationTests {
@SuppressWarnings("unchecked")
public void testBulkIndexingItemFailure() {
this.contextRunner
.withPropertyValues("elasticsearch.consumer.index=foo_" + UUID.randomUUID(), "elasticsearch.consumer.batch-size=10",
"spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress())
.withPropertyValues("elasticsearch.consumer.index=foo_" + UUID.randomUUID(),
"elasticsearch.consumer.batch-size=10",
"spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress())
.run(context -> {
final Consumer<Message<?>> elasticsearchConsumer = context.getBean("elasticsearchConsumer", Consumer.class);
final ElasticsearchConsumerProperties properties = context.getBean(ElasticsearchConsumerProperties.class);
final Consumer<Message<?>> elasticsearchConsumer = context.getBean("elasticsearchConsumer",
Consumer.class);
final ElasticsearchConsumerProperties properties = context
.getBean(ElasticsearchConsumerProperties.class);
final ElasticsearchClient elasticsearchClient = context.getBean(ElasticsearchClient.class);
for (int i = 0; i < properties.getBatchSize(); i++) {
final GetRequest getRequest = new GetRequest.Builder().index(properties.getIndex()).id(Integer.toString(i)).build();
final GetRequest getRequest = new GetRequest.Builder().index(properties.getIndex())
.id(Integer.toString(i))
.build();
assertThatExceptionOfType(ElasticsearchException.class)
.isThrownBy(() -> elasticsearchClient.get(getRequest, JsonData.class))
.withFailMessage("Expected index not found exception for message %d")
@@ -242,12 +252,13 @@ public class ElasticsearchConsumerApplicationTests {
MessageBuilder<String> builder = MessageBuilder
.withPayload("{\"seq\":" + i + ",\"age\":10,\"dateOfBirth\":1471466076564,"
+ "\"fullName\":\"John Doe\"}")
+ "\"fullName\":\"John Doe\"}")
.setHeader(ElasticsearchConsumerConfiguration.INDEX_ID_HEADER, Integer.toString(i));
if (i == 0) {
// set an invalid index name to make the first request fail
builder.setHeader(ElasticsearchConsumerConfiguration.INDEX_NAME_HEADER, "_" + properties.getIndex());
builder.setHeader(ElasticsearchConsumerConfiguration.INDEX_NAME_HEADER,
"_" + properties.getIndex());
}
final Message<String> message = builder.build();
@@ -258,8 +269,7 @@ public class ElasticsearchConsumerApplicationTests {
}
else {
// last invocation
assertThatIllegalStateException()
.isThrownBy(() -> elasticsearchConsumer.accept(message))
assertThatIllegalStateException().isThrownBy(() -> elasticsearchConsumer.accept(message))
.withMessageContaining("Bulk indexing operation completed with failures");
}
}
@@ -271,15 +281,16 @@ public class ElasticsearchConsumerApplicationTests {
public void testIndexFromMessageHeader() {
this.contextRunner
.withPropertyValues("elasticsearch.consumer.index=foo",
"spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress())
"spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress())
.run(context -> {
final Consumer<Message<?>> elasticsearchConsumer = context.getBean("elasticsearchConsumer", Consumer.class);
final ElasticsearchConsumerProperties properties = context.getBean(ElasticsearchConsumerProperties.class);
final Consumer<Message<?>> elasticsearchConsumer = context.getBean("elasticsearchConsumer",
Consumer.class);
final ElasticsearchConsumerProperties properties = context
.getBean(ElasticsearchConsumerProperties.class);
final String dynamicIndex = properties.getIndex() + "-2";
final String jsonObject = "{\"age\":10,\"dateOfBirth\":1471466076564,"
+ "\"fullName\":\"John Doe\"}";
final String jsonObject = "{\"age\":10,\"dateOfBirth\":1471466076564," + "\"fullName\":\"John Doe\"}";
final Message<String> message = MessageBuilder.withPayload(jsonObject)
.setHeader(ElasticsearchConsumerConfiguration.INDEX_ID_HEADER, "2")
.setHeader(ElasticsearchConsumerConfiguration.INDEX_NAME_HEADER, dynamicIndex)
@@ -299,16 +310,18 @@ public class ElasticsearchConsumerApplicationTests {
@SpringBootApplication
static class ElasticsearchConsumerTestApplication {
}
@Configuration
static class Config extends ElasticsearchConfiguration {
@NonNull
@Override
public ClientConfiguration clientConfiguration() {
return ClientConfiguration.builder()
.connectedTo(elasticsearch.getHttpHostAddress())
.build();
return ClientConfiguration.builder().connectedTo(elasticsearch.getHttpHostAddress()).build();
}
}
}

3
consumer/spring-file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerConfiguration.java
View File

@@ -56,8 +56,7 @@ public class FileConsumerConfiguration {
public FileWritingMessageHandler fileWritingMessageHandler(FileNameGenerator fileNameGenerator,
@Nullable ComponentCustomizer<FileWritingMessageHandler> fileWritingMessageHandlerCustomizer) {
FileWritingMessageHandler handler =
this.properties.getDirectoryExpression() != null
FileWritingMessageHandler handler = this.properties.getDirectoryExpression() != null
? new FileWritingMessageHandler(
EXPRESSION_PARSER.parseExpression(this.properties.getDirectoryExpression()))
: new FileWritingMessageHandler(this.properties.getDirectory());

4
consumer/spring-file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerProperties.java
View File

@@ -128,9 +128,7 @@ public class FileConsumerProperties {
}
public String getNameExpression() {
return (nameExpression != null)
? nameExpression + " + '" + getSuffix() + "'"
: "'" + name + getSuffix() + "'";
return (nameExpression != null) ? nameExpression + " + '" + getSuffix() + "'" : "'" + name + getSuffix() + "'";
}
public void setNameExpression(String nameExpression) {

2
consumer/spring-file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/AbstractFileConsumerTests.java
View File

@@ -54,5 +54,7 @@ public class AbstractFileConsumerTests {
@SpringBootApplication
static class FileConsumerTestApplication {
}
}

12
consumer/spring-file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/ExpressionTests.java
View File

@@ -38,12 +38,11 @@ import static org.assertj.core.api.Assertions.assertThat;
* @author Artem Bilan
* @author Soby Chacko
* <p>
* We don't need a separate SpringBootApplication for this test as there is already one available in this package.
* {@link AbstractFileConsumerTests}.
* We don't need a separate SpringBootApplication for this test as there is already one
* available in this package. {@link AbstractFileConsumerTests}.
*/
@SpringBootTest(properties = {"file.consumer.nameExpression = payload.substring(0, 4)",
"file.consumer.directoryExpression = '${java.io.tmpdir}'+'/'+headers.dir",
"file.consumer.suffix=out"})
@SpringBootTest(properties = { "file.consumer.nameExpression = payload.substring(0, 4)",
"file.consumer.directoryExpression = '${java.io.tmpdir}'+'/'+headers.dir", "file.consumer.suffix=out" })
@DirtiesContext
public class ExpressionTests {
@@ -60,6 +59,7 @@ public class ExpressionTests {
file.deleteOnExit();
assertThat(file.exists()).isTrue();
assertThat("this is something" + System.lineSeparator())
.isEqualTo(FileCopyUtils.copyToString(new FileReader(file)));
.isEqualTo(FileCopyUtils.copyToString(new FileReader(file)));
}
}

4
consumer/spring-file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/TextFileTests.java
View File

@@ -32,7 +32,7 @@ import static org.assertj.core.api.Assertions.assertThat;
* @author Artem Bilan
* @author Soby Chacko
*/
@TestPropertySource(properties = {"file.consumer.name = test", "file.consumer.suffix=txt"})
@TestPropertySource(properties = { "file.consumer.name = test", "file.consumer.suffix=txt" })
public class TextFileTests extends AbstractFileConsumerTests {
@Test
@@ -41,7 +41,7 @@ public class TextFileTests extends AbstractFileConsumerTests {
File file = new File(tempDir.toFile(), "test.txt");
assertThat(file.exists()).isTrue();
assertThat("hello file-consumer" + System.lineSeparator())
.isEqualTo(FileCopyUtils.copyToString(new FileReader(file)));
.isEqualTo(FileCopyUtils.copyToString(new FileReader(file)));
}
}

20
consumer/spring-ftp-consumer/src/main/java/org/springframework/cloud/fn/consumer/ftp/FtpConsumerConfiguration.java
View File

@@ -52,15 +52,15 @@ public class FtpConsumerConfiguration {
public IntegrationFlow ftpInboundFlow(FtpConsumerProperties properties, SessionFactory<FTPFile> ftpSessionFactory,
@Nullable ComponentCustomizer<FtpMessageHandlerSpec> ftpMessageHandlerSpecCustomizer) {
IntegrationFlowBuilder integrationFlowBuilder =
IntegrationFlow.from(MessageConsumer.class, (gateway) -> gateway.beanName("ftpConsumer"));
IntegrationFlowBuilder integrationFlowBuilder = IntegrationFlow.from(MessageConsumer.class,
(gateway) -> gateway.beanName("ftpConsumer"));
FtpMessageHandlerSpec handlerSpec =
Ftp.outboundAdapter(new FtpRemoteFileTemplate(ftpSessionFactory), properties.getMode())
.remoteDirectory(properties.getRemoteDir())
.remoteFileSeparator(properties.getRemoteFileSeparator())
.autoCreateDirectory(properties.isAutoCreateDir())
.temporaryFileSuffix(properties.getTmpFileSuffix());
FtpMessageHandlerSpec handlerSpec = Ftp
.outboundAdapter(new FtpRemoteFileTemplate(ftpSessionFactory), properties.getMode())
.remoteDirectory(properties.getRemoteDir())
.remoteFileSeparator(properties.getRemoteFileSeparator())
.autoCreateDirectory(properties.isAutoCreateDir())
.temporaryFileSuffix(properties.getTmpFileSuffix());
if (properties.getFilenameExpression() != null) {
handlerSpec.fileNameExpression(
EXPRESSION_PARSER.parseExpression(properties.getFilenameExpression()).getExpressionString());
@@ -70,9 +70,7 @@ public class FtpConsumerConfiguration {
ftpMessageHandlerSpecCustomizer.customize(handlerSpec);
}
return integrationFlowBuilder
.handle(handlerSpec)
.get();
return integrationFlowBuilder.handle(handlerSpec).get();
}
private interface MessageConsumer extends Consumer<Message<?>> {

1
consumer/spring-ftp-consumer/src/main/java/org/springframework/cloud/fn/consumer/ftp/FtpConsumerProperties.java
View File

@@ -136,4 +136,5 @@ public class FtpConsumerProperties {
public void setRemoteFileSeparator(String remoteFileSeparator) {
this.remoteFileSeparator = remoteFileSeparator;
}
}

22
consumer/spring-ftp-consumer/src/test/java/org/springframework/cloud/fn/consumer/ftp/FtpConsumerPropertiesTests.java
View File

@@ -36,8 +36,7 @@ public class FtpConsumerPropertiesTests {
@Test
public void remoteDirCanBeCustomized() {
AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext();
TestPropertyValues.of("ftp.consumer.remoteDir:/remote")
.applyTo(context);
TestPropertyValues.of("ftp.consumer.remoteDir:/remote").applyTo(context);
context.register(Conf.class);
context.refresh();
FtpConsumerProperties properties = context.getBean(FtpConsumerProperties.class);
@@ -48,8 +47,7 @@ public class FtpConsumerPropertiesTests {
@Test
public void autoCreateDirCanBeDisabled() {
AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext();
TestPropertyValues.of("ftp.consumer.autoCreateDir:false")
.applyTo(context);
TestPropertyValues.of("ftp.consumer.autoCreateDir:false").applyTo(context);
context.register(Conf.class);
context.refresh();
FtpConsumerProperties properties = context.getBean(FtpConsumerProperties.class);
@@ -60,8 +58,7 @@ public class FtpConsumerPropertiesTests {
@Test
public void tmpFileSuffixCanBeCustomized() {
AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext();
TestPropertyValues.of("ftp.consumer.tmpFileSuffix:.foo")
.applyTo(context);
TestPropertyValues.of("ftp.consumer.tmpFileSuffix:.foo").applyTo(context);
context.register(Conf.class);
context.refresh();
FtpConsumerProperties properties = context.getBean(FtpConsumerProperties.class);
@@ -72,8 +69,7 @@ public class FtpConsumerPropertiesTests {
@Test
public void tmpFileRemoteDirCanBeCustomized() {
AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext();
TestPropertyValues.of("ftp.consumer.temporaryRemoteDir:/foo")
.applyTo(context);
TestPropertyValues.of("ftp.consumer.temporaryRemoteDir:/foo").applyTo(context);
context.register(Conf.class);
context.refresh();
FtpConsumerProperties properties = context.getBean(FtpConsumerProperties.class);
@@ -84,8 +80,7 @@ public class FtpConsumerPropertiesTests {
@Test
public void remoteFileSeparatorCanBeCustomized() {
AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext();
TestPropertyValues.of("ftp.consumer.remoteFileSeparator:\\")
.applyTo(context);
TestPropertyValues.of("ftp.consumer.remoteFileSeparator:\\").applyTo(context);
context.register(Conf.class);
context.refresh();
FtpConsumerProperties properties = context.getBean(FtpConsumerProperties.class);
@@ -96,8 +91,7 @@ public class FtpConsumerPropertiesTests {
@Test
public void useTemporaryFileNameCanBeCustomized() {
AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext();
TestPropertyValues.of("ftp.consumer.useTemporaryFilename:false")
.applyTo(context);
TestPropertyValues.of("ftp.consumer.useTemporaryFilename:false").applyTo(context);
context.register(Conf.class);
context.refresh();
FtpConsumerProperties properties = context.getBean(FtpConsumerProperties.class);
@@ -108,8 +102,7 @@ public class FtpConsumerPropertiesTests {
@Test
public void fileExistsModeCanBeCustomized() {
AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext();
TestPropertyValues.of("ftp.consumer.mode:FAIL")
.applyTo(context);
TestPropertyValues.of("ftp.consumer.mode:FAIL").applyTo(context);
context.register(Conf.class);
context.refresh();
FtpConsumerProperties properties = context.getBean(FtpConsumerProperties.class);
@@ -122,4 +115,5 @@ public class FtpConsumerPropertiesTests {
static class Conf {
}
}

11
consumer/spring-ftp-consumer/src/test/java/org/springframework/cloud/fn/consumer/ftp/FtpConsumerTests.java
View File

@@ -33,13 +33,8 @@ import static org.assertj.core.api.Assertions.assertThat;
@DirtiesContext
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE,
properties = {
"ftp.consumer.remoteDir = ftpTarget",
"ftp.factory.username = foo",
"ftp.factory.password = foo",
"ftp.consumer.mode = FAIL",
"ftp.consumer.filenameExpression = payload.name.toUpperCase()"
})
properties = { "ftp.consumer.remoteDir = ftpTarget", "ftp.factory.username = foo", "ftp.factory.password = foo",
"ftp.consumer.mode = FAIL", "ftp.consumer.filenameExpression = payload.name.toUpperCase()" })
public class FtpConsumerTests extends FtpTestSupport {
@Autowired
@@ -72,5 +67,7 @@ public class FtpConsumerTests extends FtpTestSupport {
@SpringBootApplication
static class FtpConsumerTestApplication {
}
}

4
consumer/spring-jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/DefaultInitializationScriptResource.java
View File

@@ -25,8 +25,8 @@ import org.apache.commons.logging.LogFactory;
import org.springframework.core.io.ByteArrayResource;
/**
* An in-memory script crafted for dropping-creating the table we're working with.
* All columns are created as VARCHAR(2000).
* An in-memory script crafted for dropping-creating the table we're working with. All
* columns are created as VARCHAR(2000).
*
* @author Eric Bottard
* @author Thomas Risberg

41
consumer/spring-jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerConfiguration.java
View File

@@ -126,10 +126,10 @@ public class JdbcConsumerConfiguration {
@Bean
IntegrationFlow jdbcConsumerFlow(@Qualifier("aggregator") MessageHandler aggregator,
JdbcMessageHandler jdbcMessageHandler) {
JdbcMessageHandler jdbcMessageHandler) {
final IntegrationFlowBuilder builder =
IntegrationFlow.from(Consumer.class, gateway -> gateway.beanName("jdbcConsumer"));
final IntegrationFlowBuilder builder = IntegrationFlow.from(Consumer.class,
gateway -> gateway.beanName("jdbcConsumer"));
if (properties.getBatchSize() > 1 || properties.getIdleTimeout() > 0) {
builder.handle(aggregator);
}
@@ -172,8 +172,8 @@ public class JdbcConsumerConfiguration {
this.spelExpressionParser.parseExpression(qualified));
}
catch (SpelParseException e) {
logger.info("failed to parse qualified fallback expression " + qualified +
"; be sure your expression uses the 'payload.' prefix where necessary");
logger.info("failed to parse qualified fallback expression " + qualified
+ "; be sure your expression uses the 'payload.' prefix where necessary");
}
}
}
@@ -189,17 +189,17 @@ public class JdbcConsumerConfiguration {
? message.getHeaders().get(MessageHeaders.CONTENT_TYPE).toString()
: MimeTypeUtils.APPLICATION_JSON_VALUE;
if (message.getPayload() instanceof Iterable) {
Stream<Object> messageStream =
StreamSupport.stream(((Iterable<?>) message.getPayload()).spliterator(), false)
.map(payload -> {
if (payload instanceof byte[]) {
return convertibleContentType(contentType) ?
new String(((byte[]) payload)) : payload;
}
else {
return payload;
}
});
Stream<Object> messageStream = StreamSupport
.stream(((Iterable<?>) message.getPayload()).spliterator(), false)
.map(payload -> {
if (payload instanceof byte[]) {
return convertibleContentType(contentType) ? new String(((byte[]) payload))
: payload;
}
else {
return payload;
}
});
convertedMessage = new MutableMessage<>(messageStream.collect(Collectors.toList()),
message.getHeaders());
}
@@ -213,8 +213,8 @@ public class JdbcConsumerConfiguration {
super.handleMessageInternal(convertedMessage);
}
};
SqlParameterSourceFactory parameterSourceFactory =
new ParameterFactory(columnExpressionVariations, this.evaluationContext);
SqlParameterSourceFactory parameterSourceFactory = new ParameterFactory(columnExpressionVariations,
this.evaluationContext);
jdbcMessageHandler.setSqlParameterSourceFactory(parameterSourceFactory);
return jdbcMessageHandler;
}
@@ -228,9 +228,8 @@ public class JdbcConsumerConfiguration {
databasePopulator.setIgnoreFailedDrops(true);
dataSourceInitializer.setDatabasePopulator(databasePopulator);
if ("true".equals(properties.getInitialize())) {
databasePopulator.addScript(
new DefaultInitializationScriptResource(this.properties.getTableName(),
this.properties.getColumnsMap().keySet()));
databasePopulator.addScript(new DefaultInitializationScriptResource(this.properties.getTableName(),
this.properties.getColumnsMap().keySet()));
}
else {
databasePopulator.addScript(resourceLoader.getResource(this.properties.getInitialize()));

5
consumer/spring-jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerProperties.java
View File

@@ -38,8 +38,8 @@ public class JdbcConsumerProperties {
private String tableName = "messages";
/**
* The comma separated colon-based pairs of column names and SpEL expressions for values to insert/update.
* Names are used at initialization time to issue the DDL.
* The comma separated colon-based pairs of column names and SpEL expressions for
* values to insert/update. Names are used at initialization time to issue the DDL.
*/
private String columns = "payload:payload.toString()";
@@ -106,4 +106,5 @@ public class JdbcConsumerProperties {
}
return this.columnsMap;
}
}

16
consumer/spring-jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/ShorthandMapConverter.java
View File

@@ -23,12 +23,14 @@ import org.springframework.core.convert.converter.Converter;
import org.springframework.util.Assert;
/**
* A Converter from String to Map that accepts csv {@literal key:value} pairs
* (similar to what comes out of the box in Spring Core) but also simple
* {@literal key} items, in which case the value is assumed to be equal to the key.
* A Converter from String to Map that accepts csv {@literal key:value} pairs (similar to
* what comes out of the box in Spring Core) but also simple {@literal key} items, in
* which case the value is assumed to be equal to the key.
* <p>
* <p>Additionally, commas and colons can be escaped by using a backslash, which is
* useful if said mappings are to be used for SpEL for example.</p>
* <p>
* Additionally, commas and colons can be escaped by using a backslash, which is useful if
* said mappings are to be used for SpEL for example.
* </p>
*
* @author Eric Bottard
* @author Artem Bilan
@@ -49,8 +51,8 @@ public class ShorthandMapConverter implements Converter<String, Map<String, Stri
}
// Split on colon, if not preceded by backslash
String[] keyValuePair = unescaped.split("(?<!\\\\):");
Assert.isTrue(keyValuePair.length <= 2, "'" + unescaped +
"' could not be parsed to a 'key:value' pair or simple 'key' with implicit value");
Assert.isTrue(keyValuePair.length <= 2, "'" + unescaped
+ "' could not be parsed to a 'key:value' pair or simple 'key' with implicit value");
String key = keyValuePair[0].trim().replace("\\:", ":");
String value = keyValuePair.length == 2 ? keyValuePair[1].trim().replace("\\:", ":") : key;
result.put(key, value);

7
consumer/spring-jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/BatchInsertTimeoutTests.java
View File

@@ -32,7 +32,7 @@ import org.springframework.test.context.TestPropertySource;
* @author Soby Chacko
* @author Szabolcs Stremler
*/
@TestPropertySource(properties = {"jdbc.consumer.batchSize=1000", "jdbc.consumer.idleTimeout=100"})
@TestPropertySource(properties = { "jdbc.consumer.batchSize=1000", "jdbc.consumer.idleTimeout=100" })
public class BatchInsertTimeoutTests extends JdbcConsumerApplicationTests {
@Test
@@ -43,8 +43,9 @@ public class BatchInsertTimeoutTests extends JdbcConsumerApplicationTests {
final Message<Payload> message = MessageBuilder.withPayload(sent).build();
jdbcConsumer.accept(message);
}
Awaitility.await().until(() -> jdbcOperations
.queryForObject("select count(*) from messages", Integer.class), value -> value == numberOfInserts);
Awaitility.await()
.until(() -> jdbcOperations.queryForObject("select count(*) from messages", Integer.class),
value -> value == numberOfInserts);
}
}

4
consumer/spring-jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/DataReceivedAsByteArrayTests.java
View File

@@ -41,8 +41,8 @@ public class DataReceivedAsByteArrayTests extends JdbcConsumerApplicationTests {
String hello = "{\"a\": \"hello\"}";
final Message<byte[]> message = MessageBuilder.withPayload(hello.getBytes()).build();
jdbcConsumer.accept(message);
final Integer count =
jdbcOperations.queryForObject("select count(*) from messages where a = ?", Integer.class, "hello");
final Integer count = jdbcOperations.queryForObject("select count(*) from messages where a = ?", Integer.class,
"hello");
assertThat(count).isEqualTo(1);
}

10
consumer/spring-jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ExplicitTableCreationTests.java
View File

@@ -34,9 +34,8 @@ import static org.assertj.core.api.Assertions.assertThat;
* @author Soby Chacko
* @author Szabolcs Stremler
*/
@TestPropertySource(properties = {"jdbc.consumer.tableName=foobar",
"jdbc.consumer.initialize=classpath:explicit-script.sql",
"jdbc.consumer.columns=a,b"})
@TestPropertySource(properties = { "jdbc.consumer.tableName=foobar",
"jdbc.consumer.initialize=classpath:explicit-script.sql", "jdbc.consumer.columns=a,b" })
public class ExplicitTableCreationTests extends JdbcConsumerApplicationTests {
@Test
@@ -44,9 +43,8 @@ public class ExplicitTableCreationTests extends JdbcConsumerApplicationTests {
Payload sent = new Payload("hello", 42);
final Message<Payload> message = MessageBuilder.withPayload(sent).build();
jdbcConsumer.accept(message);
Payload result =
jdbcOperations.query("select a, b from foobar", new BeanPropertyRowMapper<>(Payload.class))
.get(0);
Payload result = jdbcOperations.query("select a, b from foobar", new BeanPropertyRowMapper<>(Payload.class))
.get(0);
assertThat(result).usingRecursiveComparison().isEqualTo(sent);
}

8
consumer/spring-jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/HeaderInsertTests.java
View File

@@ -39,10 +39,10 @@ public class HeaderInsertTests extends JdbcConsumerApplicationTests {
@Test
public void testHeaderInsertion() {
Payload sent = new Payload("hello", 42);
final Message<Payload> message = MessageBuilder.withPayload(sent)
.setHeader("foo", "bar").build();
final Message<Payload> message = MessageBuilder.withPayload(sent).setHeader("foo", "bar").build();
jdbcConsumer.accept(message);
assertThat(jdbcOperations.queryForObject("select count(*) from messages where a = ?",
Integer.class, "bar")).isEqualTo(1);
assertThat(jdbcOperations.queryForObject("select count(*) from messages where a = ?", Integer.class, "bar"))
.isEqualTo(1);
}
}

10
consumer/spring-jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ImplicitTableCreationTests.java
View File

@@ -34,10 +34,8 @@ import static org.assertj.core.api.Assertions.assertThat;
* @author Soby Chacko
* @author Szabolcs Stremler
*/
@TestPropertySource(properties = {
"jdbc.consumer.tableName=no_script",
"jdbc.consumer.initialize=true",
"jdbc.consumer.columns=a,b"})
@TestPropertySource(properties = { "jdbc.consumer.tableName=no_script", "jdbc.consumer.initialize=true",
"jdbc.consumer.columns=a,b" })
public class ImplicitTableCreationTests extends JdbcConsumerApplicationTests {
@Test
@@ -45,8 +43,8 @@ public class ImplicitTableCreationTests extends JdbcConsumerApplicationTests {
Payload sent = new Payload("hello", 42);
final Message<Payload> message = MessageBuilder.withPayload(sent).build();
jdbcConsumer.accept(message);
Payload result = jdbcOperations
.query("select a, b from no_script", new BeanPropertyRowMapper<>(Payload.class)).get(0);
Payload result = jdbcOperations.query("select a, b from no_script", new BeanPropertyRowMapper<>(Payload.class))
.get(0);
assertThat(result).usingRecursiveComparison().isEqualTo(sent);
}

1
consumer/spring-jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerApplicationTests.java
View File

@@ -88,6 +88,7 @@ public class JdbcConsumerApplicationTests {
@SpringBootApplication
static class JdbcConsumerTestApplication {
}
}

18
consumer/spring-jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JsonStringPayloadInsertTests.java
View File

@@ -47,15 +47,15 @@ public class JsonStringPayloadInsertTests extends JdbcConsumerApplicationTests {
jdbcConsumer.accept(message2);
final Message<String> message3 = MessageBuilder.withPayload(stringC).build();
jdbcConsumer.accept(message3);
assertThat(jdbcOperations.queryForObject(
"select count(*) from messages where a = ? and b = ?",
Integer.class, "hello1", 42)).isEqualTo(1);
assertThat(jdbcOperations.queryForObject(
"select count(*) from messages where a = ? and b IS NULL",
Integer.class, "hello2")).isEqualTo(1);
assertThat(jdbcOperations.queryForObject(
"select count(*) from messages where a = ? and b IS NULL",
Integer.class, "hello3")).isEqualTo(1);
assertThat(jdbcOperations.queryForObject("select count(*) from messages where a = ? and b = ?", Integer.class,
"hello1", 42))
.isEqualTo(1);
assertThat(jdbcOperations.queryForObject("select count(*) from messages where a = ? and b IS NULL",
Integer.class, "hello2"))
.isEqualTo(1);
assertThat(jdbcOperations.queryForObject("select count(*) from messages where a = ? and b IS NULL",
Integer.class, "hello3"))
.isEqualTo(1);
}
}

15
consumer/spring-jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/MapPayloadInsertTests.java
View File

@@ -58,12 +58,15 @@ public class MapPayloadInsertTests extends JdbcConsumerApplicationTests {
jdbcConsumer.accept(message2);
final Message<Map<String, Object>> message3 = MessageBuilder.withPayload(mapC).build();
jdbcConsumer.accept(message3);
assertThat(namedParameterJdbcOperations.queryForObject(
"select count(*) from messages where a = :a and b = :b", mapA, Integer.class)).isEqualTo(1);
assertThat(namedParameterJdbcOperations.queryForObject(
"select count(*) from messages where a = :a and b IS NULL", mapB, Integer.class)).isEqualTo(1);
assertThat(namedParameterJdbcOperations.queryForObject(
"select count(*) from messages where a = :a and b IS NULL", mapC, Integer.class)).isEqualTo(1);
assertThat(namedParameterJdbcOperations.queryForObject("select count(*) from messages where a = :a and b = :b",
mapA, Integer.class))
.isEqualTo(1);
assertThat(namedParameterJdbcOperations
.queryForObject("select count(*) from messages where a = :a and b IS NULL", mapB, Integer.class))
.isEqualTo(1);
assertThat(namedParameterJdbcOperations
.queryForObject("select count(*) from messages where a = :a and b IS NULL", mapC, Integer.class))
.isEqualTo(1);
}
}

4
consumer/spring-jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleMappingTests.java
View File

@@ -42,8 +42,8 @@ public class SimpleMappingTests extends JdbcConsumerApplicationTests {
Payload sent = new Payload("hello", 42);
final Message<Payload> message = MessageBuilder.withPayload(sent).build();
jdbcConsumer.accept(message);
Payload result = jdbcOperations
.query("select a, b from messages", new BeanPropertyRowMapper<>(Payload.class)).get(0);
Payload result = jdbcOperations.query("select a, b from messages", new BeanPropertyRowMapper<>(Payload.class))
.get(0);
assertThat(result).usingRecursiveComparison().isEqualTo(sent);
}

4
consumer/spring-jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SpELTests.java
View File

@@ -44,8 +44,8 @@ public class SpELTests extends JdbcConsumerApplicationTests {
final Message<Payload> message = MessageBuilder.withPayload(sent).build();
jdbcConsumer.accept(message);
Payload expected = new Payload("hell", 666);
Payload result = jdbcOperations
.query("select a, b from messages", new BeanPropertyRowMapper<>(Payload.class)).get(0);
Payload result = jdbcOperations.query("select a, b from messages", new BeanPropertyRowMapper<>(Payload.class))
.get(0);
assertThat(result).usingRecursiveComparison().isEqualTo(expected);
}

8
consumer/spring-jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/UnqualifiableColumnExpressionTests.java
View File

@@ -37,10 +37,12 @@ public class UnqualifiableColumnExpressionTests extends JdbcConsumerApplicationT
@Test
public void doesNotFailParsingUnqualifiableExpression() {
// if the app initializes, the test condition passes, but go ahead and apply the column expression anyway
// if the app initializes, the test condition passes, but go ahead and apply the
// column expression anyway
jdbcConsumer.accept(MessageBuilder.withPayload(new Payload("desrever", 123)).build());
assertThat(jdbcOperations.queryForObject("select count(*) from messages where a = ? and b = ?",
Integer.class, "reversed", 123)).isEqualTo(1);
assertThat(jdbcOperations.queryForObject("select count(*) from messages where a = ? and b = ?", Integer.class,
"reversed", 123))
.isEqualTo(1);
}
}

Some files were not shown because too many files have changed in this diff Show More