Upgrade Debezium to 2.3.0.Final
* address some review comments * minor pom improvements * undo Debezium BOM
This commit is contained in:
@@ -1,9 +1,9 @@
|
||||
= Debezium Auto-Configuration
|
||||
|
||||
This module provides a generic https://debezium.io/documentation/reference/2.2/development/engine.html[DebeziumEngine.Builder] auto-configuration that can be reused and composed in other applications.
|
||||
This module provides a generic https://debezium.io/documentation/reference/development/engine.html[DebeziumEngine.Builder] auto-configuration that can be reused and composed in other applications.
|
||||
|
||||
IMPORTANT: The `DebeziumEngine` does not required Kafka or Kafka Connect as it runs embedded inside your application.
|
||||
This approach though comes with some delivery guarantee limitations as explained https://debezium.io/documentation/reference/2.2/development/engine.html#%5Fhandling_failures[here].
|
||||
This approach though comes with some delivery guarantee limitations as explained https://debezium.io/documentation/reference/development/engine.html#%5Fhandling_failures[here].
|
||||
|
||||
The `Debezium Engine` is a https://en.wikipedia.org/wiki/Change_data_capture[Change Data Capture] (CDC) utility, that allows *capturing* database change events and process them with custom `java.util.Consumer` or `io.debezium.engine.ChangeConsumer` event handler implementations.
|
||||
|
||||
@@ -36,7 +36,7 @@ compile "org.springframework.cloud.fn:debezium-autoconfigure:{project-version}"
|
||||
----
|
||||
====
|
||||
|
||||
and include the https://debezium.io/documentation/reference/2.2/connectors/index.html[debezium connector] dependency for the selected Database.
|
||||
and include the https://debezium.io/documentation/reference/connectors/index.html[debezium connector] dependency for the selected Database.
|
||||
For example the postgres debezium connector dependency looks like this:
|
||||
|
||||
====
|
||||
@@ -178,29 +178,29 @@ The table below lists all available Debezium properties for each connecter.
|
||||
|===
|
||||
| Connector | Connector properties
|
||||
|
||||
|https://debezium.io/documentation/reference/2.2/connectors/mysql.html[MySQL]
|
||||
|https://debezium.io/documentation/reference/2.2/connectors/mysql.html#mysql-connector-properties
|
||||
|https://debezium.io/documentation/reference/reference/connectors/mysql.html[MySQL]
|
||||
|https://debezium.io/documentation/reference/connectors/mysql.html#mysql-connector-properties
|
||||
|
||||
|https://debezium.io/documentation/reference/2.2/connectors/mongodb.html[MongoDB]
|
||||
|https://debezium.io/documentation/reference/2.2/connectors/mongodb.html#mongodb-connector-properties
|
||||
|https://debezium.io/documentation/reference/connectors/mongodb.html[MongoDB]
|
||||
|https://debezium.io/documentation/reference/connectors/mongodb.html#mongodb-connector-properties
|
||||
|
||||
|https://debezium.io/documentation/reference/2.2/connectors/postgresql.html[PostgreSQL]
|
||||
|https://debezium.io/documentation/reference/2.2/connectors/postgresql.html#postgresql-connector-properties
|
||||
|https://debezium.io/documentation/reference/connectors/postgresql.html[PostgreSQL]
|
||||
|https://debezium.io/documentation/reference/connectors/postgresql.html#postgresql-connector-properties
|
||||
|
||||
|https://debezium.io/documentation/reference/2.2/connectors/oracle.html[Oracle]
|
||||
|https://debezium.io/documentation/reference/2.2/connectors/oracle.html#oracle-connector-properties
|
||||
|https://debezium.io/documentation/reference/connectors/oracle.html[Oracle]
|
||||
|https://debezium.io/documentation/reference/connectors/oracle.html#oracle-connector-properties
|
||||
|
||||
|https://debezium.io/documentation/reference/2.2/connectors/sqlserver.html[SQL Server]
|
||||
|https://debezium.io/documentation/reference/2.2/connectors/sqlserver.html#sqlserver-connector-properties
|
||||
|https://debezium.io/documentation/reference/connectors/sqlserver.html[SQL Server]
|
||||
|https://debezium.io/documentation/reference/connectors/sqlserver.html#sqlserver-connector-properties
|
||||
|
||||
|https://debezium.io/documentation/reference/2.2/connectors/db2.html[DB2]
|
||||
|https://debezium.io/documentation/reference/2.2/connectors/db2.html#db2-connector-properties
|
||||
|https://debezium.io/documentation/reference/connectors/db2.html[DB2]
|
||||
|https://debezium.io/documentation/reference/connectors/db2.html#db2-connector-properties
|
||||
|
||||
|https://debezium.io/documentation/reference/2.2/connectors/vitess.html[Vitess]
|
||||
|https://debezium.io/documentation/reference/2.2/connectors/vitess.html#vitess-connector-properties
|
||||
|https://debezium.io/documentation/reference/connectors/vitess.html[Vitess]
|
||||
|https://debezium.io/documentation/reference/connectors/vitess.html#vitess-connector-properties
|
||||
|
||||
|https://debezium.io/documentation/reference/2.2/connectors/spanner.html[Spanner]
|
||||
|https://debezium.io/documentation/reference/2.2/connectors/spanner.html#spanner-connector-properties
|
||||
|https://debezium.io/documentation/reference/connectors/spanner.html[Spanner]
|
||||
|https://debezium.io/documentation/reference/connectors/spanner.html#spanner-connector-properties
|
||||
|
||||
|===
|
||||
|
||||
@@ -287,7 +287,7 @@ Follow the https://debezium.io/documentation/reference/stable/transformations/ev
|
||||
|
||||
When a Debezium source runs, it reads information from the source and periodically records `offsets` that define how much of that information it has processed.
|
||||
Should the source be restarted, it will use the last recorded offset to know where in the source information it should resume reading.
|
||||
Out of the box, the following https://debezium.io/documentation/reference/2.2/development/engine.html#engine-properties[offset storage configuration] options are provided:
|
||||
Out of the box, the following https://debezium.io/documentation/reference/development/engine.html#engine-properties[offset storage configuration] options are provided:
|
||||
|
||||
==== In-Memory
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
<description>Debezium Spring Boot auto-configuration</description>
|
||||
|
||||
<properties>
|
||||
<version.debezium>2.3.0.CR1</version.debezium>
|
||||
<version.debezium>2.3.0.Final</version.debezium>
|
||||
<apicurio.version>2.4.2.Final</apicurio.version>
|
||||
</properties>
|
||||
|
||||
@@ -26,12 +26,16 @@
|
||||
<version>${version.debezium}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>slf4j-reload4j</artifactId>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-reload4j</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
@@ -46,6 +50,12 @@
|
||||
<groupId>io.apicurio</groupId>
|
||||
<artifactId>apicurio-registry-client</artifactId>
|
||||
<version>${apicurio.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<!-- TEST -->
|
||||
@@ -72,6 +82,12 @@
|
||||
<artifactId>spring-jdbc</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-classic</artifactId>
|
||||
<version>1.4.8</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<plugins>
|
||||
|
||||
@@ -22,7 +22,6 @@ import java.util.Objects;
|
||||
|
||||
import io.debezium.engine.ChangeEvent;
|
||||
import io.debezium.engine.DebeziumEngine;
|
||||
import io.debezium.engine.DebeziumEngine.Builder;
|
||||
import io.debezium.engine.DebeziumEngine.CompletionCallback;
|
||||
import io.debezium.engine.DebeziumEngine.ConnectorCallback;
|
||||
import io.debezium.engine.format.KeyValueHeaderChangeEventFormat;
|
||||
@@ -123,6 +122,7 @@ public class DebeziumEngineBuilderAutoConfiguration {
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public DebeziumEngine.Builder<ChangeEvent<byte[], byte[]>> debeziumEngineBuilder(
|
||||
OffsetCommitPolicy offsetCommitPolicy, CompletionCallback completionCallback,
|
||||
ConnectorCallback connectorCallback, DebeziumProperties properties, Clock debeziumClock) {
|
||||
|
||||
@@ -51,7 +51,6 @@ import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.test.jdbc.JdbcTestUtils;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.awaitility.Awaitility.await;
|
||||
|
||||
/**
|
||||
@@ -66,7 +65,7 @@ public class DebeziumEngineBuilderAutoConfigurationIntegrationTest {
|
||||
private static final Log logger = LogFactory.getLog(DebeziumEngineBuilderAutoConfigurationIntegrationTest.class);
|
||||
|
||||
private static final String DATABASE_NAME = "inventory";
|
||||
public static final String IMAGE_TAG = "2.2.0.Final";
|
||||
public static final String IMAGE_TAG = "2.3.0.Final";
|
||||
public static final String DEBEZIUM_EXAMPLE_MYSQL_IMAGE = "debezium/example-mysql:" + IMAGE_TAG;
|
||||
|
||||
@TempDir
|
||||
@@ -136,8 +135,7 @@ public class DebeziumEngineBuilderAutoConfigurationIntegrationTest {
|
||||
"VALUES('Test666', 'Test666', 'Test666@spring.org')");
|
||||
JdbcTestUtils.deleteFromTableWhere(jdbcTemplate, "customers", "first_name = ?", "Test666");
|
||||
|
||||
await().atMost(Duration.ofSeconds(30))
|
||||
.untilAsserted(() -> assertThat(testConsumer.recordList).hasSizeGreaterThanOrEqualTo(52));
|
||||
await().atMost(Duration.ofSeconds(30)).until(() -> (testConsumer.recordList.size() >= 52));
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
<description>Debezium Supplier</description>
|
||||
|
||||
<properties>
|
||||
<version.debezium>2.3.0.CR1</version.debezium>
|
||||
<version.debezium>2.3.0.Final</version.debezium>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
@@ -40,7 +40,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = {
|
||||
"spring.cloud.function.definition=debeziumSupplier",
|
||||
|
||||
// https://debezium.io/documentation/reference/2.2/transformations/event-flattening.html
|
||||
// https://debezium.io/documentation/reference/transformations/event-flattening.html
|
||||
"debezium.properties.transforms=unwrap",
|
||||
"debezium.properties.transforms.unwrap.type=io.debezium.transforms.ExtractNewRecordState",
|
||||
"debezium.properties.transforms.unwrap.drop.tombstones=true",
|
||||
@@ -73,7 +73,7 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
@Testcontainers
|
||||
public class DebeziumSupplierIntegrationTest {
|
||||
|
||||
public static final String IMAGE_TAG = "2.2.0.Final";
|
||||
public static final String IMAGE_TAG = "2.3.0.Final";
|
||||
public static final String DEBEZIUM_EXAMPLE_MYSQL_IMAGE = "debezium/example-mysql:" + IMAGE_TAG;
|
||||
|
||||
@Container
|
||||
@@ -106,7 +106,7 @@ public class DebeziumSupplierIntegrationTest {
|
||||
Flux<Message<?>> messageFlux = this.debeziumSupplier.get();
|
||||
|
||||
// Message size should correspond to the number of insert statements in:
|
||||
// https://github.com/debezium/container-images/blob/main/examples/mysql/2.2/inventory.sql
|
||||
// https://github.com/debezium/container-images/blob/main/examples/mysql/2.3/inventory.sql
|
||||
// filtered by Customers and Addresses table.
|
||||
StepVerifier.create(messageFlux)
|
||||
.expectNextCount(16) // Skip the DDL transaction logs.
|
||||
|
||||
Reference in New Issue
Block a user