From ca8228809acdaf6ed073e1a921a183b917a18dea Mon Sep 17 00:00:00 2001 From: Christian Tzolov Date: Tue, 27 Jun 2023 18:58:27 +0200 Subject: [PATCH] Upgrade Debezium to 2.3.0.Final * address some review comments * minor pom improvements * undo Debezium BOM --- common/debezium-autoconfigure/README.adoc | 40 +++++++++---------- common/debezium-autoconfigure/pom.xml | 22 ++++++++-- ...ebeziumEngineBuilderAutoConfiguration.java | 2 +- ...ilderAutoConfigurationIntegrationTest.java | 6 +-- supplier/debezium-supplier/pom.xml | 2 +- .../DebeziumSupplierIntegrationTest.java | 6 +-- 6 files changed, 46 insertions(+), 32 deletions(-) diff --git a/common/debezium-autoconfigure/README.adoc b/common/debezium-autoconfigure/README.adoc index 914f2831..b618e8a6 100644 --- a/common/debezium-autoconfigure/README.adoc +++ b/common/debezium-autoconfigure/README.adoc @@ -1,9 +1,9 @@ = Debezium Auto-Configuration -This module provides a generic https://debezium.io/documentation/reference/2.2/development/engine.html[DebeziumEngine.Builder] auto-configuration that can be reused and composed in other applications. +This module provides a generic https://debezium.io/documentation/reference/development/engine.html[DebeziumEngine.Builder] auto-configuration that can be reused and composed in other applications. IMPORTANT: The `DebeziumEngine` does not required Kafka or Kafka Connect as it runs embedded inside your application. -This approach though comes with some delivery guarantee limitations as explained https://debezium.io/documentation/reference/2.2/development/engine.html#%5Fhandling_failures[here]. +This approach though comes with some delivery guarantee limitations as explained https://debezium.io/documentation/reference/development/engine.html#%5Fhandling_failures[here]. The `Debezium Engine` is a https://en.wikipedia.org/wiki/Change_data_capture[Change Data Capture] (CDC) utility, that allows *capturing* database change events and process them with custom `java.util.Consumer` or `io.debezium.engine.ChangeConsumer` event handler implementations. @@ -36,7 +36,7 @@ compile "org.springframework.cloud.fn:debezium-autoconfigure:{project-version}" ---- ==== -and include the https://debezium.io/documentation/reference/2.2/connectors/index.html[debezium connector] dependency for the selected Database. +and include the https://debezium.io/documentation/reference/connectors/index.html[debezium connector] dependency for the selected Database. For example the postgres debezium connector dependency looks like this: ==== @@ -178,29 +178,29 @@ The table below lists all available Debezium properties for each connecter. |=== | Connector | Connector properties -|https://debezium.io/documentation/reference/2.2/connectors/mysql.html[MySQL] -|https://debezium.io/documentation/reference/2.2/connectors/mysql.html#mysql-connector-properties +|https://debezium.io/documentation/reference/reference/connectors/mysql.html[MySQL] +|https://debezium.io/documentation/reference/connectors/mysql.html#mysql-connector-properties -|https://debezium.io/documentation/reference/2.2/connectors/mongodb.html[MongoDB] -|https://debezium.io/documentation/reference/2.2/connectors/mongodb.html#mongodb-connector-properties +|https://debezium.io/documentation/reference/connectors/mongodb.html[MongoDB] +|https://debezium.io/documentation/reference/connectors/mongodb.html#mongodb-connector-properties -|https://debezium.io/documentation/reference/2.2/connectors/postgresql.html[PostgreSQL] -|https://debezium.io/documentation/reference/2.2/connectors/postgresql.html#postgresql-connector-properties +|https://debezium.io/documentation/reference/connectors/postgresql.html[PostgreSQL] +|https://debezium.io/documentation/reference/connectors/postgresql.html#postgresql-connector-properties -|https://debezium.io/documentation/reference/2.2/connectors/oracle.html[Oracle] -|https://debezium.io/documentation/reference/2.2/connectors/oracle.html#oracle-connector-properties +|https://debezium.io/documentation/reference/connectors/oracle.html[Oracle] +|https://debezium.io/documentation/reference/connectors/oracle.html#oracle-connector-properties -|https://debezium.io/documentation/reference/2.2/connectors/sqlserver.html[SQL Server] -|https://debezium.io/documentation/reference/2.2/connectors/sqlserver.html#sqlserver-connector-properties +|https://debezium.io/documentation/reference/connectors/sqlserver.html[SQL Server] +|https://debezium.io/documentation/reference/connectors/sqlserver.html#sqlserver-connector-properties -|https://debezium.io/documentation/reference/2.2/connectors/db2.html[DB2] -|https://debezium.io/documentation/reference/2.2/connectors/db2.html#db2-connector-properties +|https://debezium.io/documentation/reference/connectors/db2.html[DB2] +|https://debezium.io/documentation/reference/connectors/db2.html#db2-connector-properties -|https://debezium.io/documentation/reference/2.2/connectors/vitess.html[Vitess] -|https://debezium.io/documentation/reference/2.2/connectors/vitess.html#vitess-connector-properties +|https://debezium.io/documentation/reference/connectors/vitess.html[Vitess] +|https://debezium.io/documentation/reference/connectors/vitess.html#vitess-connector-properties -|https://debezium.io/documentation/reference/2.2/connectors/spanner.html[Spanner] -|https://debezium.io/documentation/reference/2.2/connectors/spanner.html#spanner-connector-properties +|https://debezium.io/documentation/reference/connectors/spanner.html[Spanner] +|https://debezium.io/documentation/reference/connectors/spanner.html#spanner-connector-properties |=== @@ -287,7 +287,7 @@ Follow the https://debezium.io/documentation/reference/stable/transformations/ev When a Debezium source runs, it reads information from the source and periodically records `offsets` that define how much of that information it has processed. Should the source be restarted, it will use the last recorded offset to know where in the source information it should resume reading. -Out of the box, the following https://debezium.io/documentation/reference/2.2/development/engine.html#engine-properties[offset storage configuration] options are provided: +Out of the box, the following https://debezium.io/documentation/reference/development/engine.html#engine-properties[offset storage configuration] options are provided: ==== In-Memory diff --git a/common/debezium-autoconfigure/pom.xml b/common/debezium-autoconfigure/pom.xml index 706fe89b..482883bf 100644 --- a/common/debezium-autoconfigure/pom.xml +++ b/common/debezium-autoconfigure/pom.xml @@ -15,7 +15,7 @@ Debezium Spring Boot auto-configuration - 2.3.0.CR1 + 2.3.0.Final 2.4.2.Final @@ -26,12 +26,16 @@ ${version.debezium} - slf4j-reload4j org.slf4j + slf4j-reload4j - slf4j-log4j12 org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-api @@ -46,6 +50,12 @@ io.apicurio apicurio-registry-client ${apicurio.version} + + + org.slf4j + slf4j-api + + @@ -72,6 +82,12 @@ spring-jdbc test + + ch.qos.logback + logback-classic + 1.4.8 + test + diff --git a/common/debezium-autoconfigure/src/main/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfiguration.java b/common/debezium-autoconfigure/src/main/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfiguration.java index 13243525..6e6c6656 100644 --- a/common/debezium-autoconfigure/src/main/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfiguration.java +++ b/common/debezium-autoconfigure/src/main/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfiguration.java @@ -22,7 +22,6 @@ import java.util.Objects; import io.debezium.engine.ChangeEvent; import io.debezium.engine.DebeziumEngine; -import io.debezium.engine.DebeziumEngine.Builder; import io.debezium.engine.DebeziumEngine.CompletionCallback; import io.debezium.engine.DebeziumEngine.ConnectorCallback; import io.debezium.engine.format.KeyValueHeaderChangeEventFormat; @@ -123,6 +122,7 @@ public class DebeziumEngineBuilderAutoConfiguration { } @Bean + @ConditionalOnMissingBean public DebeziumEngine.Builder> debeziumEngineBuilder( OffsetCommitPolicy offsetCommitPolicy, CompletionCallback completionCallback, ConnectorCallback connectorCallback, DebeziumProperties properties, Clock debeziumClock) { diff --git a/common/debezium-autoconfigure/src/test/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfigurationIntegrationTest.java b/common/debezium-autoconfigure/src/test/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfigurationIntegrationTest.java index b029ee6b..fd39520f 100644 --- a/common/debezium-autoconfigure/src/test/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfigurationIntegrationTest.java +++ b/common/debezium-autoconfigure/src/test/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfigurationIntegrationTest.java @@ -51,7 +51,6 @@ import org.springframework.context.annotation.Primary; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.test.jdbc.JdbcTestUtils; -import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; /** @@ -66,7 +65,7 @@ public class DebeziumEngineBuilderAutoConfigurationIntegrationTest { private static final Log logger = LogFactory.getLog(DebeziumEngineBuilderAutoConfigurationIntegrationTest.class); private static final String DATABASE_NAME = "inventory"; - public static final String IMAGE_TAG = "2.2.0.Final"; + public static final String IMAGE_TAG = "2.3.0.Final"; public static final String DEBEZIUM_EXAMPLE_MYSQL_IMAGE = "debezium/example-mysql:" + IMAGE_TAG; @TempDir @@ -136,8 +135,7 @@ public class DebeziumEngineBuilderAutoConfigurationIntegrationTest { "VALUES('Test666', 'Test666', 'Test666@spring.org')"); JdbcTestUtils.deleteFromTableWhere(jdbcTemplate, "customers", "first_name = ?", "Test666"); - await().atMost(Duration.ofSeconds(30)) - .untilAsserted(() -> assertThat(testConsumer.recordList).hasSizeGreaterThanOrEqualTo(52)); + await().atMost(Duration.ofSeconds(30)).until(() -> (testConsumer.recordList.size() >= 52)); }); } diff --git a/supplier/debezium-supplier/pom.xml b/supplier/debezium-supplier/pom.xml index 4059fac9..f55e6829 100644 --- a/supplier/debezium-supplier/pom.xml +++ b/supplier/debezium-supplier/pom.xml @@ -15,7 +15,7 @@ Debezium Supplier - 2.3.0.CR1 + 2.3.0.Final diff --git a/supplier/debezium-supplier/src/test/java/org/springframework/cloud/fn/supplier/debezium/it/supplier/DebeziumSupplierIntegrationTest.java b/supplier/debezium-supplier/src/test/java/org/springframework/cloud/fn/supplier/debezium/it/supplier/DebeziumSupplierIntegrationTest.java index 58655f23..eacb1ef4 100644 --- a/supplier/debezium-supplier/src/test/java/org/springframework/cloud/fn/supplier/debezium/it/supplier/DebeziumSupplierIntegrationTest.java +++ b/supplier/debezium-supplier/src/test/java/org/springframework/cloud/fn/supplier/debezium/it/supplier/DebeziumSupplierIntegrationTest.java @@ -40,7 +40,7 @@ import static org.assertj.core.api.Assertions.assertThat; @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { "spring.cloud.function.definition=debeziumSupplier", - // https://debezium.io/documentation/reference/2.2/transformations/event-flattening.html + // https://debezium.io/documentation/reference/transformations/event-flattening.html "debezium.properties.transforms=unwrap", "debezium.properties.transforms.unwrap.type=io.debezium.transforms.ExtractNewRecordState", "debezium.properties.transforms.unwrap.drop.tombstones=true", @@ -73,7 +73,7 @@ import static org.assertj.core.api.Assertions.assertThat; @Testcontainers public class DebeziumSupplierIntegrationTest { - public static final String IMAGE_TAG = "2.2.0.Final"; + public static final String IMAGE_TAG = "2.3.0.Final"; public static final String DEBEZIUM_EXAMPLE_MYSQL_IMAGE = "debezium/example-mysql:" + IMAGE_TAG; @Container @@ -106,7 +106,7 @@ public class DebeziumSupplierIntegrationTest { Flux> messageFlux = this.debeziumSupplier.get(); // Message size should correspond to the number of insert statements in: - // https://github.com/debezium/container-images/blob/main/examples/mysql/2.2/inventory.sql + // https://github.com/debezium/container-images/blob/main/examples/mysql/2.3/inventory.sql // filtered by Customers and Addresses table. StepVerifier.create(messageFlux) .expectNextCount(16) // Skip the DDL transaction logs.