diff --git a/README.adoc b/README.adoc index 0de8a947..af4891cd 100644 --- a/README.adoc +++ b/README.adoc @@ -73,7 +73,7 @@ The following are the four major components of this repository. |link:functions/supplier/websocket-supplier/README.adoc[Websocket] | |link:functions/consumer/twitter-consumer/README.adoc[Twitter] -| +||link:functions/supplier/cdc-debezium-supplier/README.adoc[CDC Debezium] | |link:functions/consumer/websocket-consumer/README.adoc[Websocket] | @@ -140,7 +140,7 @@ The following are the four major components of this repository. |link:applications/source/websocket-source/README.adoc[Websocket] | |link:applications/sink/twitter-update-sink/README.adoc[Twitter Update] -| +||link:applications/source/cdc-debezium-source/README.adoc[CDC Debezium] | |link:applications/sink/wavefront-sink/README.adoc[Wavefront] |=== diff --git a/applications/source/cdc-debezium-source/README.adoc b/applications/source/cdc-debezium-source/README.adoc new file mode 100644 index 00000000..ddc9a2f8 --- /dev/null +++ b/applications/source/cdc-debezium-source/README.adoc @@ -0,0 +1,289 @@ +//tag::ref-doc[] += CDC Source + +https://en.wikipedia.org/wiki/Change_data_capture[Change Data Capture] (CDC) `source` that captures and streams change events from various databases. +Currently, it supports `MySQL`, `PostgreSQL`, `MongoDB`, `Oracle` and `SQL Server` databases. + +Build upon https://debezium.io/docs/embedded/[Debezium Embedded Connector], the `CDC Source` allows capturing and streaming database changes over different message binders such Apache Kafka, RabbitMQ and all Spring Cloud Stream supporter brokers. + +It supports all Debezium configuration properties. Just add the `cdc.config.` prefix to the existing Debezium properties. For example to set the Debezium's `connector.class` property use the `cdc.config.connector.class` source property instead. + +We provide convenient shortcuts for the most frequently used Debezium properties. For example instead of the long `cdc.config.connector.class=io.debezium.connector.mysql.MySqlConnector` Debezium property you can use our `cdc.connector=mysql` shortcut. The table below lists all available shortcuts along with the Debezium properties they represent. +The Debezium properties (e.g. `cdc.config.XXX`) always have precedence over the shortcuts! + +The CDC Source introduces a new default `BackingOffsetStore` configuration, based on the [MetadataStore](https://github.com/spring-cloud/stream-applications/tree/master/functions/common/metadata-store-common) service. Later provides various microservices friendly ways for storing the offset metadata. + +== Options + +//tag::configuration-properties[] +$$cdc.config$$:: $$Spring pass-trough wrapper for debezium configuration properties. All properties with a 'cdc.config.' prefix are native Debezium properties. The prefix is removed, converting them into Debezium io.debezium.config.Configuration.$$ *($$Map$$, default: `$$$$`)* +$$cdc.connector$$:: $$Shortcut for the cdc.config.connector.class property. Either of those can be used as long as they do not contradict with each other.$$ *($$ConnectorType$$, default: `$$$$`, possible values: `mysql`,`postgres`,`mongodb`,`oracle`,`sqlserver`)* +$$cdc.flattering.add-fields$$:: $$Comma separated list of metadata fields to add to the flattened message. The fields will be prefixed with "__" or "__[<]struct]__", depending on the specification of the struct.$$ *($$String$$, default: `$$$$`)* +$$cdc.flattering.add-headers$$:: $$Comma separated list specify a list of metadata fields to add to the header of the flattened message. The fields will be prefixed with "__" or "__[struct]__".$$ *($$String$$, default: `$$$$`)* +$$cdc.flattering.delete-handling-mode$$:: $$Options for handling deleted records: (1) none - pass the records through, (2) drop - remove the records and (3) rewrite - add a '__deleted' field to the records.$$ *($$DeleteHandlingMode$$, default: `$$$$`, possible values: `drop`,`rewrite`,`none`)* +$$cdc.flattering.drop-tombstones$$:: $$By default Debezium generates tombstone records to enable Kafka compaction on deleted records. The dropTombstones can suppress the tombstone records.$$ *($$Boolean$$, default: `$$true$$`)* +$$cdc.flattering.enabled$$:: $$Enable flattering the source record events (https://debezium.io/docs/configuration/event-flattening).$$ *($$Boolean$$, default: `$$true$$`)* +$$cdc.name$$:: $$Unique name for this sourceConnector instance.$$ *($$String$$, default: `$$$$`)* +$$cdc.offset.commit-timeout$$:: $$Maximum number of milliseconds to wait for records to flush and partition offset data to be committed to offset storage before cancelling the process and restoring the offset data to be committed in a future attempt.$$ *($$Duration$$, default: `$$5000ms$$`)* +$$cdc.offset.flush-interval$$:: $$Interval at which to try committing offsets. The default is 1 minute.$$ *($$Duration$$, default: `$$60000ms$$`)* +$$cdc.offset.policy$$:: $$Offset storage commit policy.$$ *($$OffsetPolicy$$, default: `$$$$`)* +$$cdc.offset.storage$$:: $$Kafka connector tracks the number processed records and regularly stores the count (as "offsets") in a preconfigured metadata storage. On restart the connector resumes the reading from the last recorded source offset.$$ *($$OffsetStorageType$$, default: `$$$$`, possible values: `memory`,`file`,`kafka`,`metadata`)* +$$cdc.schema$$:: $$Include the schema's as part of the outbound message.$$ *($$Boolean$$, default: `$$false$$`)* +$$cdc.stream.header.convert-connect-headers$$:: $$When true the {@link org.apache.kafka.connect.header.Header} are converted into message headers with the {@link org.apache.kafka.connect.header.Header#key()} as name and {@link org.apache.kafka.connect.header.Header#value()}.$$ *($$Boolean$$, default: `$$true$$`)* +$$cdc.stream.header.offset$$:: $$Serializes the source record's offset metadata into the outbound message header under cdc.offset.$$ *($$Boolean$$, default: `$$false$$`)* +//end::configuration-properties[] + +==== Debezium property Shortcut mapping + +The table below lists all available shortcuts along with the Debezium properties they represent. + +.Table Shortcut Properties Mapping +|=== +| Shortcut | Original | Description + +|cdc.connector +|cdc.config.connector.class +|`mysql` : MySqlConnector, `postgres` : PostgresConnector, `mongodb` : MongodbSourceConnector, `oracle` : OracleConnector, `sqlserver` : SqlServerConnector + +|cdc.name +|cdc.config.name +| + +|cdc.offset.flush-interval +|cdc.config.offset.flush.interval.ms +| + +|cdc.offset.commit-timeout +|cdc.config.offset.flush.timeout.ms +| + +|cdc.offset.policy +|cdc.config.offset.commit.policy +|`periodic` : PeriodicCommitOffsetPolicy, `always` : AlwaysCommitOffsetPolicy + +|cdc.offset.storage +|cdc.config.offset.storage +|`metadata` : MetadataStoreOffsetBackingStore, `file` : FileOffsetBackingStore, `kafka` : KafkaOffsetBackingStore, `memory` : MemoryOffsetBackingStore + +|cdc.flattering.drop-tombstones +|cdc.config.drop.tombstones +| + +|cdc.flattering.delete-handling-mode +|cdc.config.delete.handling.mode +|`none` : none, `drop` : drop, `rewrite` : rewrite + +|=== + +== Database Support + +The `CDC Source` uses the Debezium utilities, and currently supports CDC for five datastores: `MySQL`, `PostgreSQL`, `MongoDB`, `Oracle` and `SQL Server` databases. + +== Examples and Testing + +The [CdcSourceIntegrationTest](), [CdcDeleteHandlingIntegrationTest]() and [CdcFlatteringIntegrationTest]() integration tests use test databases fixtures, running on the local machine. +We use pre-build debezium docker database images. +The Maven builds create the test databases fixtures with the help of the `docker-maven-plugin`. + +To run and debug the tests from your IDE you need to deploy the required database images from the command line. +Instructions below explains how to run pre-configured test databases form Docker images. + +==== MySQL + +Start the `debezium/example-mysql` in a docker: +[source, bash] +---- +docker run -it --rm --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=debezium -e MYSQL_USER=mysqluser -e MYSQL_PASSWORD=mysqlpw debezium/example-mysql:1.0 +---- + +[TIP] +==== +(optional) Use `mysql` client to connected to the database and to create a `debezium` user with required credentials: +[source, bash] +---- +docker run -it --rm --name mysqlterm --link mysql --rm mysql:5.7 sh -c 'exec mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD"' +mysql> GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'debezium' IDENTIFIED BY 'dbz'; +---- +==== + +Use following properties to connect the CDC Source to the MySQL DB: + +[source] +---- +cdc.connector=mysql # <1> + +cdc.name=my-sql-connector # <2> +cdc.config.database.server.id=85744 # <2> +cdc.config.database.server.name=my-app-connector # <2> + +cdc.config.database.user=debezium # <3> +cdc.config.database.password=dbz # <3> +cdc.config.database.hostname=localhost # <3> +cdc.config.database.port=3306 # <3> + +cdc.schema=true # <4> +cdc.flattering.enabled=true # <5> +---- +<1> Configures the CDC Source to use https://debezium.io/docs/connectors/mysql/[MySqlConnector]. (equivalent to setting `cdc.config.connector.class=io.debezium.connector.mysql.MySqlConnector`). +<2> Metadata used to identify and dispatch the incoming events. +<3> Connection to the MySQL server running on `localhost:3306` as `debezium` user. +<4> Includes the https://debezium.io/docs/connectors/mysql/#change-events-value[Change Event Value] schema in the `SourceRecord` events. +<5> Enables the https://debezium.io/docs/configuration/event-flattening/[CDC Event Flattering]. + +You can run also the `CdcSourceIntegrationTests#CdcMysqlTests` using this mysql configuration. + +==== PostgreSQL + +Start a pre-configured postgres server from the `debezium/example-postgres:1.0` Docker image: +[source, bash] +---- +docker run -it --rm --name postgres -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres debezium/example-postgres:1.0 +---- + +You can connect to this server like this: +[source, bash] +---- +psql -U postgres -h localhost -p 5432 +---- + +Use following properties to connect the CDC Source to the PostgreSQL: + +[source] +---- +cdc.connector=postgres # <1> +cdc.offset.storage=memory #<2> + +cdc.name=my-sql-connector # <3> +cdc.config.database.server.id=85744 # <3> +cdc.config.database.server.name=my-app-connector # <3> + +cdc.config.database.user=postgres # <4> +cdc.config.database.password=postgres # <4> +cdc.config.database..dbname=postgres # <4> +cdc.config.database.hostname=localhost # <4> +cdc.config.database.port=5432 # <4> + +cdc.schema=true # <5> +cdc.flattering.enabled=true # <6> +---- + +<1> Configures `CDC Source` to use https://debezium.io/docs/connectors/postgresql/[PostgresConnector]. Equivalent for setting `cdc.config.connector.class=io.debezium.connector.postgresql.PostgresConnector`. +<2> Configures the Debezium engine to use `memory` (e.g. `cdc.config.offset.storage=org.apache.kafka.connect.storage.MemoryOffsetBackingStore) backing offset store. +<3> Metadata used to identify and dispatch the incoming events. +<4> Connection to the PostgreSQL server running on `localhost:5432` as `postgres` user. +<5> Includes the https://debezium.io/docs/connectors/mysql/#change-events-value[Change Event Value] schema in the `SourceRecord` events. +<6> Enables the https://debezium.io/docs/configuration/event-flattening/[CDC Event Flattering]. + +You can run also the `CdcSourceIntegrationTests#CdcPostgresTests` using this mysql configuration. + +==== MongoDB + +Start a pre-configured mongodb from the `debezium/example-mongodb:0.10` Docker image: +[source, bash] +---- +docker run -it --rm --name mongodb -p 27017:27017 -e MONGODB_USER=debezium -e MONGODB_PASSWORD=dbz debezium/example-mongodb:0.10 +---- + +Initialize the inventory collections +[source, bash] +---- +docker exec -it mongodb sh -c 'bash -c /usr/local/bin/init-inventory.sh' +---- + +In the `mongodb` terminal output, search for a log entry like `host: "3f95a8a6516e:27017"` : +[source, bash] +---- +2019-01-10T13:46:10.004+0000 I COMMAND [conn1] command local.oplog.rs appName: "MongoDB Shell" command: replSetInitiate { replSetInitiate: { _id: "rs0", members: [ { _id: 0.0, host: "3f95a8a6516e:27017" } ] }, lsid: { id: UUID("5f477a16-d80d-41f2-9ab4-4ebecea46773") }, $db: "admin" } numYields:0 reslen:22 locks:{ Global: { acquireCount: { r: 36, w: 20, W: 2 }, acquireWaitCount: { W: 1 }, timeAcquiringMicros: { W: 312 } }, Database: { acquireCount: { r: 6, w: 4, W: 16 } }, Collection: { acquireCount: { r: 4, w: 2 } }, oplog: { acquireCount: { r: 2, w: 3 } } } protocol:op_msg 988ms +---- + +Add `127.0.0.1 3f95a8a6516e` entry to your `/etc/hosts` + + +Use following properties to connect the CDC Source to the MongoDB: + +[source] +---- +cdc.connector=mongodb # <1> +cdc.offset.storage=memory #<2> + +cdc.config.mongodb.hosts=rs0/localhost:27017 # <3> +cdc.config.mongodb.name=dbserver1 # <3> +cdc.config.mongodb.user=debezium # <3> +cdc.config.mongodb.password=dbz # <3> +cdc.config.database.whitelist=inventory # <3> + +cdc.config.tasks.max=1 # <4> + +cdc.schema=true # <5> +cdc.flattering.enabled=true # <6> +---- + +<1> Configures `CDC Source` to use https://debezium.io/docs/connectors/mongodb/[MongoDB Connector]. This maps into `cdc.config.connector.class=io.debezium.connector.mongodb.MongodbSourceConnector`. +<2> Configures the Debezium engine to use `memory` (e.g. `cdc.config.offset.storage=org.apache.kafka.connect.storage.MemoryOffsetBackingStore) backing offset store. +<3> Connection to the MongoDB running on `localhost:27017` as `debezium` user. +<4> https://debezium.io/docs/connectors/mongodb/#tasks +<5> Includes the https://debezium.io/docs/connectors/mysql/#change-events-value[Change Event Value] schema in the `SourceRecord` events. +<6> Enables the https://debezium.io/docs/configuration/event-flattening/[CDC Event Flattering]. + +You can run also the `CdcSourceIntegrationTests#CdcPostgresTests` using this mysql configuration. + +==== SQL Server + +Start a `sqlserver` from the `debezium/example-postgres:1.0` Docker image: +[source, bash] +---- +docker run -it --rm --name sqlserver -p 1433:1433 -e ACCEPT_EULA=Y -e MSSQL_PID=Standard -e SA_PASSWORD=Password! -e MSSQL_AGENT_ENABLED=true microsoft/mssql-server-linux:2017-CU9-GDR2 +---- + +Populate with sample data form debezium's sqlserver tutorial: +[source, bash] +---- +wget https://raw.githubusercontent.com/debezium/debezium-examples/master/tutorial/debezium-sqlserver-init/inventory.sql +cat ./inventory.sql | docker exec -i sqlserver bash -c '/opt/mssql-tools/bin/sqlcmd -U sa -P $SA_PASSWORD' +---- + +Use following properties to connect the CDC Source to the SQLServer: + +[source] +---- +cdc.connector=sqlserver # <1> +cdc.offset.storage=memory #<2> + +cdc.name=my-sql-connector # <3> +cdc.config.database.server.id=85744 # <3> +cdc.config.database.server.name=my-app-connector # <3> + +cdc.config.database.user=sa # <4> +cdc.config.database.password=Password! # <4> +cdc.config.database..dbname=testDB # <4> +cdc.config.database.hostname=localhost # <4> +cdc.config.database.port=1433 # <4> +---- + +<1> Configures `CDC Source` to use https://debezium.io/docs/connectors/sqlserver/[SqlServerConnector]. Equivalent for setting `cdc.config.connector.class=io.debezium.connector.sqlserver.SqlServerConnector`. +<2> Configures the Debezium engine to use `memory` (e.g. `cdc.config.offset.storage=org.apache.kafka.connect.storage.MemoryOffsetBackingStore) backing offset store. +<3> Metadata used to identify and dispatch the incoming events. +<4> Connection to the SQL Server running on `localhost:1433` as `sa` user. + +You can run also the `CdcSourceIntegrationTests#CdcSqlServerTests` using this mysql configuration. + +==== Oracle + +Start Oracle reachable from localhost and set up with the configuration, users and grants described in the https://github.com/debezium/oracle-vagrant-box[Debezium Vagrant set-up] + +Populate with sample data form Debezium's Oracle tutorial: +[source, bash] +---- +wget https://raw.githubusercontent.com/debezium/debezium-examples/master/tutorial/debezium-with-oracle-jdbc/init/inventory.sql +cat ./inventory.sql | docker exec -i dbz_oracle sqlplus debezium/dbz@//localhost:1521/ORCLPDB1 +---- + +//end::ref-doc[] + + +== Run standalone + +``` +java -jar cdc-debezium-source.jar --cdc.connector=mysql --cdc.name=my-sql-connector --cdc.config.database.server.id=85744 --cdc.config.database.server.name=my-app-connector --cdc.config.database.user=debezium --cdc.config.database.password=dbz --cdc.config.database.hostname=localhost --cdc.config.database.port=3306 --cdc.schema=true --cdc.flattering.enabled=true +``` + diff --git a/applications/source/cdc-debezium-source/pom.xml b/applications/source/cdc-debezium-source/pom.xml new file mode 100644 index 00000000..f00d2030 --- /dev/null +++ b/applications/source/cdc-debezium-source/pom.xml @@ -0,0 +1,298 @@ + + + 4.0.0 + cdc-debezium-source + 3.0.0-SNAPSHOT + cdc-debezium-source + CDC Debezium source apps + jar + + + org.springframework.cloud.stream.app + stream-applications-core + 3.0.0-SNAPSHOT + + + + + + org.springframework.cloud.fn + cdc-debezium-supplier + ${java-functions.version} + + + slf4j-log4j12 + org.slf4j + + + + + org.springframework.cloud.fn + function-test-support + ${java-functions.version} + test + + + ch.qos.logback + logback-classic + + + + + org.springframework.kafka + spring-kafka + + + org.springframework + spring-jdbc + test + + + net.javacrumbs.json-unit + json-unit + 1.25.1 + test + + + org.springframework.cloud + spring-cloud-stream-binder-kafka + + 3.0.8.BUILD-SNAPSHOT + test + + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + cdc-debezium + source + ${project.version} + org.springframework.cloud.fn.supplier.cdc.CdcSupplierConfiguration.class + cdcSupplier + + + + org.springframework.cloud.fn + cdc-debebzium-supplier + + + + + + io.fabric8 + docker-maven-plugin + 0.33.0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + mysql + debezium/example-mysql:1.0 + + + debezium + mysqluser + mysqlpw + + + 3306:3306 + + + port: 3306 + + + + + + postgres + debezium/example-postgres:1.0 + + + postgres + postgres + + + 5432:5432 + + + PostgreSQL init process complete + + + + + + + properties + docker + override + + %a/sqlserver-example:${project.version} + + ${project.basedir}/src/test/docker/sqlserver + @ + + + + Y + Standard + Password! + true + + + 1433:1433 + + + 1 rows affected + + + + + + + + + start + + generate-test-resources + + build + start + + + + stop + post-integration-test + + stop + + + + stop-pre + clean + + stop + + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + false + + spring-releases + Spring Releases + https://repo.spring.io/release + + + + false + + spring-libs-release + Spring Libs Release + https://repo.spring.io/libs-release + + + + false + + spring-milestone-release + Spring Milestone Release + https://repo.spring.io/libs-milestone + + + + + spring-releases + Spring Releases + https://repo.spring.io/libs-release + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + + + diff --git a/applications/source/cdc-debezium-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/source/cdc-debezium-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..38208648 --- /dev/null +++ b/applications/source/cdc-debezium-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -0,0 +1,5 @@ +configuration-properties.classes=org.springframework.cloud.fn.supplier.cdc.CdcSupplierProperties, \ + org.springframework.cloud.fn.supplier.cdc.CdcSupplierProperties$Header, \ + org.springframework.cloud.fn.common.cdc.CdcCommonProperties, \ + org.springframework.cloud.fn.common.cdc.CdcCommonProperties$Flattering, \ + org.springframework.cloud.fn.common.cdc.CdcCommonProperties$Offset diff --git a/applications/source/cdc-debezium-source/src/main/resources/application.properties b/applications/source/cdc-debezium-source/src/main/resources/application.properties new file mode 100644 index 00000000..8fb76f9d --- /dev/null +++ b/applications/source/cdc-debezium-source/src/main/resources/application.properties @@ -0,0 +1,2 @@ +spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration +spring.cloud.stream.kafka.default.producer.messageKeyExpression=headers['cdc_key'] diff --git a/applications/source/cdc-debezium-source/src/test/docker/mongodb/Dockerfile b/applications/source/cdc-debezium-source/src/test/docker/mongodb/Dockerfile new file mode 100644 index 00000000..e31f0af7 --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/docker/mongodb/Dockerfile @@ -0,0 +1,16 @@ +FROM debezium/example-mongodb:0.10 + +## Bundle data source +COPY entrypoint.sh /usr/local/bin/ + +# Grant permissions for the import-data script to be executable +RUN chmod +x /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/init-inventory.sh + +CMD /usr/local/bin/entrypoint.sh +#CMD ["mongod", "--replSet", "rs0", "--auth"] +#CMD ["/bin/bash", "./entrypoint.sh"] + +#CMD ["mongod", "--replSet", "rs0", "--auth", "&", "sleep", "10s", "&", "/usr/local/bin/init-inventory.sh"] +#CMD mongod --replSet rs0 --auth + diff --git a/applications/source/cdc-debezium-source/src/test/docker/mongodb/entrypoint.sh b/applications/source/cdc-debezium-source/src/test/docker/mongodb/entrypoint.sh new file mode 100644 index 00000000..24d8f8d0 --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/docker/mongodb/entrypoint.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +mongod --replSet rs0 --auth & sleep 10s diff --git a/applications/source/cdc-debezium-source/src/test/docker/mongodb/import-data.sh b/applications/source/cdc-debezium-source/src/test/docker/mongodb/import-data.sh new file mode 100644 index 00000000..529c0d44 --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/docker/mongodb/import-data.sh @@ -0,0 +1,5 @@ +#wait for the SQL Server to come up +sleep 10s + +#populate the db +/usr/local/bin/init-inventory.sh diff --git a/applications/source/cdc-debezium-source/src/test/docker/sqlserver/Dockerfile b/applications/source/cdc-debezium-source/src/test/docker/sqlserver/Dockerfile new file mode 100644 index 00000000..75f022b7 --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/docker/sqlserver/Dockerfile @@ -0,0 +1,13 @@ +FROM microsoft/mssql-server-linux:2017-CU9-GDR2 + +## Create sql directory +RUN mkdir -p /usr/src/data +WORKDIR /usr/src/data + +## Bundle data source +COPY . /usr/src/data + +# Grant permissions for the import-data script to be executable +RUN chmod +x /usr/src/data/import-data.sh + +CMD /bin/bash ./entrypoint.sh diff --git a/applications/source/cdc-debezium-source/src/test/docker/sqlserver/entrypoint.sh b/applications/source/cdc-debezium-source/src/test/docker/sqlserver/entrypoint.sh new file mode 100644 index 00000000..09c60289 --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/docker/sqlserver/entrypoint.sh @@ -0,0 +1,6 @@ +# Start SQL Server, start the script to create the DB and import the data. +# Use the tail to keep the container running +/opt/mssql/bin/sqlservr & \ +/usr/src/data/import-data.sh & \ +echo "Data Loaded" & \ +tail -f /dev/null diff --git a/applications/source/cdc-debezium-source/src/test/docker/sqlserver/import-data.sh b/applications/source/cdc-debezium-source/src/test/docker/sqlserver/import-data.sh new file mode 100644 index 00000000..374070bc --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/docker/sqlserver/import-data.sh @@ -0,0 +1,5 @@ +#wait for the SQL Server to come up +sleep 10s + +#run the setup script to create the DB and the schema in the DB +/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P $SA_PASSWORD -d master -i inventory.sql diff --git a/applications/source/cdc-debezium-source/src/test/docker/sqlserver/inventory.sql b/applications/source/cdc-debezium-source/src/test/docker/sqlserver/inventory.sql new file mode 100644 index 00000000..ae5839d1 --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/docker/sqlserver/inventory.sql @@ -0,0 +1,84 @@ +-- Create the test database +CREATE DATABASE testDB; +GO +USE testDB; +EXEC sys.sp_cdc_enable_db; + +-- Create and populate our products using a single insert with many rows +CREATE TABLE products ( + id INTEGER IDENTITY(101,1) NOT NULL PRIMARY KEY, + name VARCHAR(255) NOT NULL, + description VARCHAR(512), + weight FLOAT +); +INSERT INTO products(name,description,weight) + VALUES ('scooter','Small 2-wheel scooter',3.14); +INSERT INTO products(name,description,weight) + VALUES ('car battery','12V car battery',8.1); +INSERT INTO products(name,description,weight) + VALUES ('12-pack drill bits','12-pack of drill bits with sizes ranging from #40 to #3',0.8); +INSERT INTO products(name,description,weight) + VALUES ('hammer','12oz carpenter''s hammer',0.75); +INSERT INTO products(name,description,weight) + VALUES ('hammer','14oz carpenter''s hammer',0.875); +INSERT INTO products(name,description,weight) + VALUES ('hammer','16oz carpenter''s hammer',1.0); +INSERT INTO products(name,description,weight) + VALUES ('rocks','box of assorted rocks',5.3); +INSERT INTO products(name,description,weight) + VALUES ('jacket','water resistent black wind breaker',0.1); +INSERT INTO products(name,description,weight) + VALUES ('spare tire','24 inch spare tire',22.2); +EXEC sys.sp_cdc_enable_table @source_schema = 'dbo', @source_name = 'products', @role_name = NULL, @supports_net_changes = 0; +-- Create and populate the products on hand using multiple inserts +CREATE TABLE products_on_hand ( + product_id INTEGER NOT NULL PRIMARY KEY, + quantity INTEGER NOT NULL, + FOREIGN KEY (product_id) REFERENCES products(id) +); +INSERT INTO products_on_hand VALUES (101,3); +INSERT INTO products_on_hand VALUES (102,8); +INSERT INTO products_on_hand VALUES (103,18); +INSERT INTO products_on_hand VALUES (104,4); +INSERT INTO products_on_hand VALUES (105,5); +INSERT INTO products_on_hand VALUES (106,0); +INSERT INTO products_on_hand VALUES (107,44); +INSERT INTO products_on_hand VALUES (108,2); +INSERT INTO products_on_hand VALUES (109,5); +EXEC sys.sp_cdc_enable_table @source_schema = 'dbo', @source_name = 'products_on_hand', @role_name = NULL, @supports_net_changes = 0; +-- Create some customers ... +CREATE TABLE customers ( + id INTEGER IDENTITY(1001,1) NOT NULL PRIMARY KEY, + first_name VARCHAR(255) NOT NULL, + last_name VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL UNIQUE +); +INSERT INTO customers(first_name,last_name,email) + VALUES ('Sally','Thomas','sally.thomas@acme.com'); +INSERT INTO customers(first_name,last_name,email) + VALUES ('George','Bailey','gbailey@foobar.com'); +INSERT INTO customers(first_name,last_name,email) + VALUES ('Edward','Walker','ed@walker.com'); +INSERT INTO customers(first_name,last_name,email) + VALUES ('Anne','Kretchmar','annek@noanswer.org'); +EXEC sys.sp_cdc_enable_table @source_schema = 'dbo', @source_name = 'customers', @role_name = NULL, @supports_net_changes = 0; +-- Create some very simple orders +CREATE TABLE orders ( + id INTEGER IDENTITY(10001,1) NOT NULL PRIMARY KEY, + order_date DATE NOT NULL, + purchaser INTEGER NOT NULL, + quantity INTEGER NOT NULL, + product_id INTEGER NOT NULL, + FOREIGN KEY (purchaser) REFERENCES customers(id), + FOREIGN KEY (product_id) REFERENCES products(id) +); +INSERT INTO orders(order_date,purchaser,quantity,product_id) + VALUES ('16-JAN-2016', 1001, 1, 102); +INSERT INTO orders(order_date,purchaser,quantity,product_id) + VALUES ('17-JAN-2016', 1002, 2, 105); +INSERT INTO orders(order_date,purchaser,quantity,product_id) + VALUES ('19-FEB-2016', 1002, 2, 106); +INSERT INTO orders(order_date,purchaser,quantity,product_id) + VALUES ('21-FEB-2016', 1003, 1, 107); +EXEC sys.sp_cdc_enable_table @source_schema = 'dbo', @source_name = 'orders', @role_name = NULL, @supports_net_changes = 0; +GO diff --git a/applications/source/cdc-debezium-source/src/test/java/org/springframework/cloud/stream/app/source/cdc/CdcDeleteHandlingIntegrationTest.java b/applications/source/cdc-debezium-source/src/test/java/org/springframework/cloud/stream/app/source/cdc/CdcDeleteHandlingIntegrationTest.java new file mode 100644 index 00000000..d092ad4f --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/java/org/springframework/cloud/stream/app/source/cdc/CdcDeleteHandlingIntegrationTest.java @@ -0,0 +1,162 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.source.cdc; + +import java.time.Duration; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.springframework.boot.test.context.FilteredClassLoader; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.boot.test.context.runner.ContextConsumer; +import org.springframework.cloud.fn.common.cdc.CdcCommonProperties; +import org.springframework.cloud.stream.binder.test.OutputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.context.ApplicationContext; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.kafka.support.KafkaNull; +import org.springframework.messaging.Message; +import org.springframework.test.jdbc.JdbcTestUtils; +import org.springframework.util.ClassUtils; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.cloud.fn.supplier.cdc.CdcSupplierConfiguration.ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL; +import static org.springframework.cloud.stream.app.source.cdc.CdcTestUtils.receiveAll; + +/** + * @author Christian Tzolov + */ +public class CdcDeleteHandlingIntegrationTest { + + private final JdbcTemplate jdbcTemplate = CdcTestUtils.jdbcTemplate( + "com.mysql.cj.jdbc.Driver", + "jdbc:mysql://localhost:3306/inventory", + "root", "debezium"); + + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withUserConfiguration( + TestChannelBinderConfiguration.getCompleteConfiguration(TestCdcSourceApplication.class)) + .withPropertyValues( + "spring.cloud.stream.function.definition=cdcSupplier", + "cdc.name=my-sql-connector", + "cdc.schema=false", + "cdc.flattering.enabled=true", + "cdc.stream.header.offset=true", + "cdc.connector=mysql", + "cdc.config.database.user=debezium", + "cdc.config.database.password=dbz", + "cdc.config.database.hostname=localhost", + "cdc.config.database.port=3306", + "cdc.config.database.server.id=85744", + "cdc.config.database.server.name=my-app-connector", + "cdc.config.database.history=io.debezium.relational.history.MemoryDatabaseHistory"); + + @Test + public void handleRecordDeletionTest() { + contextRunner.withPropertyValues("cdc.flattering.deleteHandlingMode=none", "cdc.flattering.dropTombstones=true") + .run(consumer); + contextRunner.withPropertyValues("cdc.flattering.deleteHandlingMode=none", "cdc.flattering.dropTombstones=true") + .withClassLoader(new FilteredClassLoader(KafkaNull.class)) // Remove Kafka from the classpath + .run(consumer); + + contextRunner.withPropertyValues("cdc.flattering.deleteHandlingMode=none", "cdc.flattering.dropTombstones=false") + .run(consumer); + contextRunner.withPropertyValues("cdc.flattering.deleteHandlingMode=none", "cdc.flattering.dropTombstones=false") + .withClassLoader(new FilteredClassLoader(KafkaNull.class)) // Remove Kafka from the classpath + .run(consumer); + + contextRunner.withPropertyValues("cdc.flattering.deleteHandlingMode=drop", "cdc.flattering.dropTombstones=true") + .run(consumer); + contextRunner.withPropertyValues("cdc.flattering.deleteHandlingMode=drop", "cdc.flattering.dropTombstones=true") + .withClassLoader(new FilteredClassLoader(KafkaNull.class)) // Remove Kafka from the classpath + .run(consumer); + + contextRunner.withPropertyValues("cdc.flattering.deleteHandlingMode=drop", "cdc.flattering.dropTombstones=false") + .run(consumer); + contextRunner.withPropertyValues("cdc.flattering.deleteHandlingMode=drop", "cdc.flattering.dropTombstones=false") + .withClassLoader(new FilteredClassLoader(KafkaNull.class)) // Remove Kafka from the classpath + .run(consumer); + + contextRunner.withPropertyValues("cdc.flattering.deleteHandlingMode=rewrite", "cdc.flattering.dropTombstones=true") + .run(consumer); + contextRunner.withPropertyValues("cdc.flattering.deleteHandlingMode=rewrite", "cdc.flattering.dropTombstones=true") + .withClassLoader(new FilteredClassLoader(KafkaNull.class)) // Remove Kafka from the classpath + .run(consumer); + + contextRunner.withPropertyValues("cdc.flattering.deleteHandlingMode=rewrite", "cdc.flattering.dropTombstones=false") + .run(consumer); + contextRunner.withPropertyValues("cdc.flattering.deleteHandlingMode=rewrite", "cdc.flattering.dropTombstones=false") + .withClassLoader(new FilteredClassLoader(KafkaNull.class)) // Remove Kafka from the classpath + .run(consumer); + } + + final ContextConsumer consumer = context -> { + OutputDestination outputDestination = context.getBean(OutputDestination.class); + + CdcCommonProperties props = context.getBean(CdcCommonProperties.class); + boolean isKafkaPresent = ClassUtils.isPresent(ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL, context.getClassLoader()); + + CdcCommonProperties.DeleteHandlingMode deleteHandlingMode = props.getFlattering().getDeleteHandlingMode(); + boolean isDropTombstones = props.getFlattering().isDropTombstones(); + + jdbcTemplate.update("insert into `customers`(`first_name`,`last_name`,`email`) VALUES('Test666', 'Test666', 'Test666@spring.org')"); + String newRecordId = jdbcTemplate.query("select * from `customers` where `first_name` = ?", + (rs, rowNum) -> rs.getString("id"), "Test666").iterator().next(); + + List> messages = receiveAll(outputDestination); + assertThat(messages).hasSize(53); + + JdbcTestUtils.deleteFromTableWhere(jdbcTemplate, "customers", "first_name = ?", "Test666"); + + Message received; + + if (deleteHandlingMode == CdcCommonProperties.DeleteHandlingMode.drop) { + // Do nothing + } + else if (deleteHandlingMode == CdcCommonProperties.DeleteHandlingMode.none) { + received = outputDestination.receive(Duration.ofSeconds(10).toMillis()); + assertThat(received).isNotNull(); + assertThat(received.getPayload()).isEqualTo("null".getBytes()); + } + else if (deleteHandlingMode == CdcCommonProperties.DeleteHandlingMode.rewrite) { + received = outputDestination.receive(Duration.ofSeconds(10).toMillis()); + assertThat(received).isNotNull(); + assertThat(toString(received.getPayload()).contains("\"__deleted\":\"true\"")); + } + + if (!isDropTombstones && isKafkaPresent) { + received = outputDestination.receive(Duration.ofSeconds(10).toMillis()); + assertThat(received).isNotNull(); + //Tombstones event should have KafkaNull payload + assertThat(received.getPayload().getClass().getCanonicalName()) + .isEqualTo(ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL); + + String key = (String) received.getHeaders().get("cdc_key"); + //Tombstones event should carry the deleted record id in the cdc_key header + assertThat(key).isEqualTo("{\"id\":" + newRecordId + "}"); + } + + received = outputDestination.receive(Duration.ofSeconds(10).toMillis()); + assertThat(received).isNull(); + }; + + private String toString(Object object) { + return new String((byte[]) object); + } + +} diff --git a/applications/source/cdc-debezium-source/src/test/java/org/springframework/cloud/stream/app/source/cdc/CdcFlatteringIntegrationTest.java b/applications/source/cdc-debezium-source/src/test/java/org/springframework/cloud/stream/app/source/cdc/CdcFlatteringIntegrationTest.java new file mode 100644 index 00000000..4db44654 --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/java/org/springframework/cloud/stream/app/source/cdc/CdcFlatteringIntegrationTest.java @@ -0,0 +1,244 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.source.cdc; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.springframework.boot.test.context.FilteredClassLoader; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.boot.test.context.runner.ContextConsumer; +import org.springframework.cloud.fn.common.cdc.CdcCommonProperties; +import org.springframework.cloud.stream.binder.test.OutputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.context.ApplicationContext; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.kafka.support.KafkaNull; +import org.springframework.messaging.Message; +import org.springframework.test.jdbc.JdbcTestUtils; +import org.springframework.util.ClassUtils; +import org.springframework.util.StringUtils; + +import static net.javacrumbs.jsonunit.JsonAssert.assertJsonEquals; +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.cloud.fn.supplier.cdc.CdcSupplierConfiguration.ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL; +import static org.springframework.cloud.stream.app.source.cdc.CdcTestUtils.receiveAll; +import static org.springframework.cloud.stream.app.source.cdc.CdcTestUtils.resourceToString; + +/** + * @author Christian Tzolov + */ +public class CdcFlatteringIntegrationTest { + + private final JdbcTemplate jdbcTemplate = CdcTestUtils.jdbcTemplate( + "com.mysql.cj.jdbc.Driver", + "jdbc:mysql://localhost:3306/inventory", + "root", "debezium"); + + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withUserConfiguration( + TestChannelBinderConfiguration.getCompleteConfiguration(TestCdcSourceApplication.class)) + .withPropertyValues( + "spring.cloud.stream.function.definition=cdcSupplier", + "cdc.name=my-sql-connector", + "cdc.schema=false", + + "cdc.stream.header.offset=false", + + "cdc.connector=mysql", + "cdc.config.database.user=debezium", + "cdc.config.database.password=dbz", + "cdc.config.database.hostname=localhost", + "cdc.config.database.port=3306", + "cdc.config.database.server.id=85744", + "cdc.config.database.server.name=my-app-connector", + "cdc.config.database.history=io.debezium.relational.history.MemoryDatabaseHistory" + ); + + @Test + public void noFlatteredResponseNoKafka() { + contextRunner.withPropertyValues("cdc.flattering.enabled=false") + .withClassLoader(new FilteredClassLoader(KafkaNull.class)) // Remove Kafka from the classpath + .run(noFlatteringTest); + } + + @Test + public void noFlatteredResponseWithKafka() { + contextRunner.withPropertyValues("cdc.flattering.enabled=false") + .run(noFlatteringTest); + } + + final ContextConsumer noFlatteringTest = context -> { + OutputDestination outputDestination = context.getBean(OutputDestination.class); + boolean isKafkaPresent = ClassUtils.isPresent(ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL, context.getClassLoader()); + + List> messages = receiveAll(outputDestination); + assertThat(messages).hasSize(52); + + assertJsonEquals(resourceToString( + "classpath:/json/mysql_ddl_drop_inventory_address_table.json"), + toString(messages.get(1).getPayload())); + assertThat(messages.get(1).getHeaders().get("cdc_topic")).isEqualTo("my-app-connector"); + assertJsonEquals("{\"databaseName\":\"inventory\"}", messages.get(1).getHeaders().get("cdc_key")); + + assertJsonEquals(resourceToString("classpath:/json/mysql_insert_inventory_products_106.json"), + toString(messages.get(39).getPayload())); + assertThat(messages.get(39).getHeaders().get("cdc_topic")).isEqualTo("my-app-connector.inventory.products"); + assertJsonEquals("{\"id\":106}", messages.get(39).getHeaders().get("cdc_key")); + + jdbcTemplate.update( + "insert into `customers`(`first_name`,`last_name`,`email`) VALUES('Test666', 'Test666', 'Test666@spring.org')"); + String newRecordId = jdbcTemplate.query("select * from `customers` where `first_name` = ?", + (rs, rowNum) -> rs.getString("id"), "Test666").iterator().next(); + jdbcTemplate.update("UPDATE `customers` SET `last_name`='Test999' WHERE first_name = 'Test666'"); + JdbcTestUtils.deleteFromTableWhere(jdbcTemplate, "customers", "first_name = ?", "Test666"); + + messages = receiveAll(outputDestination); + + assertThat(messages).hasSize(isKafkaPresent ? 4 : 3); + + assertJsonEquals(resourceToString("classpath:/json/mysql_update_inventory_customers.json"), + toString(messages.get(1).getPayload())); + assertThat(messages.get(1).getHeaders().get("cdc_topic")).isEqualTo("my-app-connector.inventory.customers"); + assertJsonEquals("{\"id\":" + newRecordId + "}", messages.get(1).getHeaders().get("cdc_key")); + + assertJsonEquals(resourceToString("classpath:/json/mysql_delete_inventory_customers.json"), + toString(messages.get(2).getPayload())); + assertThat(messages.get(1).getHeaders().get("cdc_topic")).isEqualTo("my-app-connector.inventory.customers"); + + assertJsonEquals("{\"id\":" + newRecordId + "}", messages.get(1).getHeaders().get("cdc_key")); + + if (isKafkaPresent) { + assertThat(messages.get(3).getPayload().getClass().getCanonicalName()) + .isEqualTo(ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL, + "Tombstones event should have KafkaNull payload"); + assertThat(messages.get(3).getHeaders().get("cdc_topic")) + .isEqualTo("my-app-connector.inventory.customers"); + assertJsonEquals("{\"id\":" + newRecordId + "}", messages.get(3).getHeaders().get("cdc_key")); + } + }; + + @Test + public void flatteredResponseNoKafka() { + contextRunner.withPropertyValues( + "cdc.flattering.enabled=true", + "cdc.flattering.deleteHandlingMode=none", + "cdc.flattering.dropTombstones=false", + "cdc.flattering.addHeaders=op", + "cdc.flattering.addFields=name,db") + .withClassLoader(new FilteredClassLoader(KafkaNull.class)) // Remove Kafka from the classpath + .run(flatteringTest); + } + + @Test + public void flatteredResponseWithKafka() { + contextRunner.withPropertyValues( + "cdc.flattering.enabled=true", + "cdc.flattering.deleteHandlingMode=none", + "cdc.flattering.dropTombstones=false", + "cdc.flattering.addHeaders=op", + "cdc.flattering.addFields=name,db") + .run(flatteringTest); + } + + @Test + public void flatteredResponseWithKafkaDropTombstone() { + contextRunner.withPropertyValues( + "cdc.flattering.enabled=true", + "cdc.flattering.deleteHandlingMode=none", + "cdc.flattering.dropTombstones=true", + "cdc.flattering.addHeaders=op", + "cdc.flattering.addFields=name,db") + .run(flatteringTest); + } + + final ContextConsumer flatteringTest = context -> { + OutputDestination outputDestination = context.getBean(OutputDestination.class); + boolean isKafkaPresent = ClassUtils.isPresent(ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL, context.getClassLoader()); + + List> messages = receiveAll(outputDestination); + assertThat(messages).hasSize(52); + + CdcCommonProperties.Flattering flatteringProps = context.getBean(CdcCommonProperties.class).getFlattering(); + + assertJsonEquals(resourceToString( + "classpath:/json/mysql_ddl_drop_inventory_address_table.json"), + toString(messages.get(1).getPayload())); + assertThat(messages.get(1).getHeaders().get("cdc_topic")).isEqualTo("my-app-connector"); + assertJsonEquals("{\"databaseName\":\"inventory\"}", + messages.get(1).getHeaders().get("cdc_key")); + + if (flatteringProps.isEnabled()) { + assertJsonEquals(resourceToString("classpath:/json/mysql_flattered_insert_inventory_products_106.json"), + toString(messages.get(39).getPayload())); + } + else { + assertJsonEquals(resourceToString("classpath:/json/mysql_insert_inventory_products_106.json"), + toString(messages.get(39).getPayload())); + } + assertThat(messages.get(39).getHeaders().get("cdc_topic")).isEqualTo("my-app-connector.inventory.products"); + assertJsonEquals("{\"id\":106}", messages.get(39).getHeaders().get("cdc_key")); + + if (flatteringProps.isEnabled() && flatteringProps.getAddHeaders().contains("op")) { + assertThat(messages.get(39).getHeaders().get("__op")).isEqualTo("c"); + } + + jdbcTemplate.update( + "insert into `customers`(`first_name`,`last_name`,`email`) VALUES('Test666', 'Test666', 'Test666@spring.org')"); + String newRecordId = jdbcTemplate.query("select * from `customers` where `first_name` = ?", + (rs, rowNum) -> rs.getString("id"), "Test666").iterator().next(); + jdbcTemplate.update("UPDATE `customers` SET `last_name`='Test999' WHERE first_name = 'Test666'"); + JdbcTestUtils.deleteFromTableWhere(jdbcTemplate, "customers", "first_name = ?", "Test666"); + + messages = receiveAll(outputDestination); + + assertThat(messages).hasSize((!flatteringProps.isDropTombstones() && isKafkaPresent) ? 4 : 3); + + assertJsonEquals(resourceToString("classpath:/json/mysql_flattered_update_inventory_customers.json"), + toString(messages.get(1).getPayload())); + + assertThat(messages.get(1).getHeaders().get("cdc_topic")).isEqualTo("my-app-connector.inventory.customers"); + assertJsonEquals("{\"id\":" + newRecordId + "}", messages.get(1).getHeaders().get("cdc_key")); + if (!StringUtils.isEmpty(flatteringProps.getAddHeaders()) && flatteringProps.getAddHeaders().contains("op")) { + assertThat(messages.get(1).getHeaders().get("__op")).isEqualTo("u"); + } + + if (flatteringProps.getDeleteHandlingMode() == CdcCommonProperties.DeleteHandlingMode.none) { + assertThat(toString(messages.get(2).getPayload())).isEqualTo("null"); + assertThat(messages.get(1).getHeaders().get("cdc_topic")).isEqualTo("my-app-connector.inventory.customers"); + assertJsonEquals("{\"id\":" + newRecordId + "}", messages.get(1).getHeaders().get("cdc_key")); + if (!StringUtils.isEmpty(flatteringProps.getAddHeaders()) && flatteringProps.getAddHeaders().contains("op")) { + assertThat(messages.get(2).getHeaders().get("__op")).isEqualTo("d"); + } + } + + if (!flatteringProps.isDropTombstones() && isKafkaPresent) { + assertThat(messages.get(3).getPayload().getClass().getCanonicalName()) + .isEqualTo(ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL, + "Tombstones event should have KafkaNull payload"); + assertThat(messages.get(3).getHeaders().get("cdc_topic")) + .isEqualTo("my-app-connector.inventory.customers"); + assertJsonEquals("{\"id\":" + newRecordId + "}", messages.get(3).getHeaders().get("cdc_key")); + } + }; + + private String toString(Object object) { + return new String((byte[]) object); + } + +} diff --git a/applications/source/cdc-debezium-source/src/test/java/org/springframework/cloud/stream/app/source/cdc/CdcSourceDatabasesIntegrationTest.java b/applications/source/cdc-debezium-source/src/test/java/org/springframework/cloud/stream/app/source/cdc/CdcSourceDatabasesIntegrationTest.java new file mode 100644 index 00000000..55a76b8a --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/java/org/springframework/cloud/stream/app/source/cdc/CdcSourceDatabasesIntegrationTest.java @@ -0,0 +1,123 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.source.cdc; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.stream.binder.test.OutputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.messaging.Message; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.cloud.stream.app.source.cdc.CdcTestUtils.receiveAll; + +/** + * @author Christian Tzolov + */ +public class CdcSourceDatabasesIntegrationTest { + + private final SpringApplicationBuilder applicationBuilder = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(TestCdcSourceApplication.class)) + .web(WebApplicationType.NONE) + .properties("spring.cloud.stream.function.definition=cdcSupplier", + "cdc.name=my-sql-connector", + "cdc.flattering.dropTombstones=false", + "cdc.schema=false", + "cdc.flattering.enabled=true", + "cdc.stream.header.offset=true", + + "cdc.config.database.server.id=85744", + "cdc.config.database.server.name=my-app-connector", + "cdc.config.database.history=io.debezium.relational.history.MemoryDatabaseHistory"); + + @Test + public void mysql() { + try (ConfigurableApplicationContext context = applicationBuilder + .run("--cdc.connector=mysql", + "--cdc.config.database.user=debezium", + "--cdc.config.database.password=dbz", + "--cdc.config.database.hostname=localhost", + "--cdc.config.database.port=3306")) { + OutputDestination outputDestination = context.getBean(OutputDestination.class); + // Using local region here + List> messages = receiveAll(outputDestination); + assertThat(messages).isNotNull(); + assertThat(messages).hasSize(52); + } + } + + @Test + public void sqlServer() { + + try (ConfigurableApplicationContext context = applicationBuilder + .run("--cdc.connector=sqlserver", + //"--cdc.config.database.user=Standard", + "--cdc.config.database.user=sa", + "--cdc.config.database.password=Password!", + "--cdc.config.database.dbname=testDB", + + "--cdc.config.database.hostname=localhost", + "--cdc.config.database.port=1433" + )) { + OutputDestination outputDestination = context.getBean(OutputDestination.class); + // Using local region here + List> messages = receiveAll(outputDestination); + assertThat(messages).isNotNull(); + assertThat(messages).hasSize(30); + } + } + + @Test + public void postgres() { + try (ConfigurableApplicationContext context = applicationBuilder + .run("--cdc.connector=postgres", + "--cdc.config.database.user=postgres", + "--cdc.config.database.password=postgres", + "--cdc.config.database.dbname=postgres", + "--cdc.config.database.hostname=localhost", + "--cdc.config.database.port=5432")) { + OutputDestination outputDestination = context.getBean(OutputDestination.class); + // Using local region here + List> messages = receiveAll(outputDestination); + assertThat(messages).isNotNull(); + assertThat(messages).hasSize(5786); + } + } + + //@Test + public void mongodb() { + try (ConfigurableApplicationContext context = applicationBuilder + .run("--cdc.connector=mongodb", + "--cdc.config.tasks.max=1", + "--cdc.config.mongodb.hosts=rs0/localhost:27017", + "--cdc.config.mongodb.name=dbserver1", + "--cdc.config.mongodb.user=debezium", + "--cdc.config.mongodb.password=dbz", + "--cdc.config.database.whitelist=inventory")) { + OutputDestination outputDestination = context.getBean(OutputDestination.class); + // Using local region here + List> messages = receiveAll(outputDestination); + assertThat(messages).isNotNull(); + assertThat(messages).hasSize(666); + } + } +} diff --git a/applications/source/cdc-debezium-source/src/test/java/org/springframework/cloud/stream/app/source/cdc/CdcTestUtils.java b/applications/source/cdc-debezium-source/src/test/java/org/springframework/cloud/stream/app/source/cdc/CdcTestUtils.java new file mode 100644 index 00000000..76a513a2 --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/java/org/springframework/cloud/stream/app/source/cdc/CdcTestUtils.java @@ -0,0 +1,68 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.source.cdc; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +import org.springframework.cloud.stream.binder.test.OutputDestination; +import org.springframework.core.io.DefaultResourceLoader; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.springframework.messaging.Message; +import org.springframework.util.StreamUtils; + +/** + * @author Christian Tzolov + */ +public final class CdcTestUtils { + + private CdcTestUtils() { + + } + + public static JdbcTemplate jdbcTemplate(String jdbcDriver, String jdbcUrl, String user, String password) { + + DriverManagerDataSource dataSource = new DriverManagerDataSource(); + dataSource.setDriverClassName(jdbcDriver); + dataSource.setUrl(jdbcUrl); + dataSource.setUsername(user); + dataSource.setPassword(password); + + return new JdbcTemplate(dataSource); + } + + public static List> receiveAll(OutputDestination outputDestination) { + List> list = new ArrayList<>(); + Message received; + do { + received = outputDestination.receive(Duration.ofSeconds(20).toMillis()); + if (received != null) { + list.add(received); + } + } while (received != null); + return list; + } + + public static String resourceToString(String resourcePath) throws IOException { + return StreamUtils.copyToString( + new DefaultResourceLoader().getResource(resourcePath).getInputStream(), StandardCharsets.UTF_8); + } +} diff --git a/applications/source/cdc-debezium-source/src/test/java/org/springframework/cloud/stream/app/source/cdc/TestCdcSourceApplication.java b/applications/source/cdc-debezium-source/src/test/java/org/springframework/cloud/stream/app/source/cdc/TestCdcSourceApplication.java new file mode 100644 index 00000000..fb7d46e3 --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/java/org/springframework/cloud/stream/app/source/cdc/TestCdcSourceApplication.java @@ -0,0 +1,45 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.source.cdc; + +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; +import org.springframework.cloud.fn.supplier.cdc.CdcSupplierConfiguration; +import org.springframework.cloud.stream.binder.kafka.KafkaNullConverter; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.messaging.converter.MessageConverter; + +/** + * @author Christian Tzolov + */ +@SpringBootConfiguration +@EnableAutoConfiguration(exclude = MongoAutoConfiguration.class) +@Import(CdcSupplierConfiguration.class) +public class TestCdcSourceApplication { + + /** + * This is only required for testing. In production the KafkaNullConverter bean is auto-configured. + */ + @Bean + @ConditionalOnMissingBean(KafkaNullConverter.class) + MessageConverter kafkaNullConverter() { + return new KafkaNullConverter(); + } +} diff --git a/applications/source/cdc-debezium-source/src/test/resources/json/mysql_ddl_drop_inventory_address_table.json b/applications/source/cdc-debezium-source/src/test/resources/json/mysql_ddl_drop_inventory_address_table.json new file mode 100644 index 00000000..bdcd6c7e --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/resources/json/mysql_ddl_drop_inventory_address_table.json @@ -0,0 +1,20 @@ +{ + "source": { + "version": "${json-unit.ignore}", + "connector": "mysql", + "name": "my-app-connector", + "ts_ms": 0, + "snapshot": "true", + "db": "inventory", + "table": "addresses", + "server_id": 0, + "gtid": null, + "file": "${json-unit.ignore}", + "pos": "${json-unit.ignore}", + "row": 0, + "thread": null, + "query": null + }, + "databaseName": "inventory", + "ddl": "DROP TABLE IF EXISTS `inventory`.`addresses`" +} diff --git a/applications/source/cdc-debezium-source/src/test/resources/json/mysql_delete_inventory_customers.json b/applications/source/cdc-debezium-source/src/test/resources/json/mysql_delete_inventory_customers.json new file mode 100644 index 00000000..090b2f3c --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/resources/json/mysql_delete_inventory_customers.json @@ -0,0 +1,28 @@ +{ + "before": { + "id": "${json-unit.ignore}", + "first_name": "Test666", + "last_name": "Test999", + "email": "Test666@spring.org" + }, + "after": null, + "source": { + "version": "${json-unit.ignore}", + "connector": "mysql", + "name": "my-app-connector", + "ts_ms": "${json-unit.ignore}", + "snapshot": "false", + "db": "inventory", + "table": "customers", + "server_id": "${json-unit.ignore}", + "gtid": null, + "file": "${json-unit.ignore}", + "pos": "${json-unit.ignore}", + "row": 0, + "thread": "${json-unit.ignore}", + "query": null + }, + "op": "d", + "ts_ms": "${json-unit.ignore}", + "transaction": null +} diff --git a/applications/source/cdc-debezium-source/src/test/resources/json/mysql_flattered_insert_inventory_products_106.json b/applications/source/cdc-debezium-source/src/test/resources/json/mysql_flattered_insert_inventory_products_106.json new file mode 100644 index 00000000..abf55d0b --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/resources/json/mysql_flattered_insert_inventory_products_106.json @@ -0,0 +1,8 @@ +{ + "__db" : "inventory", + "__name" : "my-app-connector", + "id": 106, + "name": "hammer", + "description": "16oz carpenter's hammer", + "weight": 1.0 +} diff --git a/applications/source/cdc-debezium-source/src/test/resources/json/mysql_flattered_update_inventory_customers.json b/applications/source/cdc-debezium-source/src/test/resources/json/mysql_flattered_update_inventory_customers.json new file mode 100644 index 00000000..cbf43ac5 --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/resources/json/mysql_flattered_update_inventory_customers.json @@ -0,0 +1,8 @@ +{ + "id": "${json-unit.ignore}", + "first_name": "Test666", + "last_name": "Test999", + "email": "Test666@spring.org", + "__name": "my-app-connector", + "__db": "inventory" +} diff --git a/applications/source/cdc-debezium-source/src/test/resources/json/mysql_insert_inventory_products_106.json b/applications/source/cdc-debezium-source/src/test/resources/json/mysql_insert_inventory_products_106.json new file mode 100644 index 00000000..363f33c3 --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/resources/json/mysql_insert_inventory_products_106.json @@ -0,0 +1,28 @@ +{ + "before": null, + "after": { + "id": 106, + "name": "hammer", + "description": "16oz carpenter's hammer", + "weight": 1.0 + }, + "source": { + "version": "${json-unit.ignore}", + "connector": "mysql", + "name": "my-app-connector", + "ts_ms": 0, + "snapshot": "true", + "db": "inventory", + "table": "products", + "server_id": 0, + "gtid": null, + "file": "${json-unit.ignore}", + "pos": "${json-unit.ignore}", + "row": 0, + "thread": null, + "query": null + }, + "op": "c", + "ts_ms": "${json-unit.ignore}", + "transaction": null +} diff --git a/applications/source/cdc-debezium-source/src/test/resources/json/mysql_update_inventory_customers.json b/applications/source/cdc-debezium-source/src/test/resources/json/mysql_update_inventory_customers.json new file mode 100644 index 00000000..f1a35f88 --- /dev/null +++ b/applications/source/cdc-debezium-source/src/test/resources/json/mysql_update_inventory_customers.json @@ -0,0 +1,33 @@ +{ + "before": { + "id": "${json-unit.ignore}", + "first_name": "Test666", + "last_name": "Test666", + "email": "Test666@spring.org" + }, + "after": { + "id": "${json-unit.ignore}", + "first_name": "Test666", + "last_name": "Test999", + "email": "Test666@spring.org" + }, + "source": { + "version": "${json-unit.ignore}", + "connector": "mysql", + "name": "my-app-connector", + "ts_ms": "${json-unit.ignore}", + "snapshot": "false", + "db": "inventory", + "table": "customers", + "server_id": "${json-unit.ignore}", + "gtid": null, + "file": "${json-unit.ignore}", + "pos": "${json-unit.ignore}", + "row": 0, + "thread": "${json-unit.ignore}", + "query": null + }, + "op": "u", + "ts_ms": "${json-unit.ignore}", + "transaction": null +} diff --git a/applications/source/pom.xml b/applications/source/pom.xml index 980a5caf..38bdcfc0 100644 --- a/applications/source/pom.xml +++ b/applications/source/pom.xml @@ -28,5 +28,6 @@ twitter-search-source twitter-message-source websocket-source + cdc-debezium-source diff --git a/applications/stream-applications-build/pom.xml b/applications/stream-applications-build/pom.xml index 0b905cea..82159be2 100644 --- a/applications/stream-applications-build/pom.xml +++ b/applications/stream-applications-build/pom.xml @@ -20,6 +20,7 @@ 3.0.0-SNAPSHOT + ${apps.version} ${apps.version} ${apps.version} ${apps.version} diff --git a/applications/stream-applications-build/stream-applications-descriptor/pom.xml b/applications/stream-applications-build/stream-applications-descriptor/pom.xml index 850f945a..e80d1102 100644 --- a/applications/stream-applications-build/stream-applications-descriptor/pom.xml +++ b/applications/stream-applications-build/stream-applications-descriptor/pom.xml @@ -43,6 +43,8 @@ pom.properties['repo-spring-io']= "${jdbc-source.version}".contains('SNAPSHOT') ? 'repo.spring.io/snapshot' : "${jdbc-source.version}".contains('-') ? 'repo.spring.io/milestone' : 'repo.spring.io/release' + pom.properties['cdc-debezium-source-docker.tag']= + "${cdc-debezium-source.version}".contains('SNAPSHOT') ? 'latest' : "${cdc-debezium-source.version}" pom.properties['file-source-docker.tag']= "${file-source.version}".contains('SNAPSHOT') ? 'latest' : "${file-source.version}" pom.properties['ftp-source-docker.tag']= diff --git a/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/kafka-apps-docker.properties b/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/kafka-apps-docker.properties index 6e2c5a7b..35700caf 100644 --- a/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/kafka-apps-docker.properties +++ b/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/kafka-apps-docker.properties @@ -1,3 +1,4 @@ +source.cdc-debezium=docker:springcloudstream/cdc-debezium-source-kafka:@cdc-debezium-source-docker.tag@ source.file=docker:springcloudstream/file-source-kafka:@file-source-docker.tag@ source.ftp=docker:springcloudstream/ftp-source-kafka:@ftp-source-docker.tag@ source.geode=docker:springcloudstream/geode-source-kafka:@geode-source-docker.tag@ diff --git a/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/kafka-apps-maven-repo-url.properties b/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/kafka-apps-maven-repo-url.properties index 29397e9b..50de05ca 100644 --- a/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/kafka-apps-maven-repo-url.properties +++ b/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/kafka-apps-maven-repo-url.properties @@ -1,3 +1,5 @@ +source.cdc-debezium=https://@repo-spring-io@/org/springframework/cloud/stream/app/cdc-debezium-source-kafka/@cdc-debezium-source.version@/cdc-debezium-source-kafka-@cdc-debezium-source.version@.jar +source.cdc-debezium.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/cdc-debezium-source-kafka/@cdc-debezium-source.version@/cdc-debezium-source-kafka-@cdc-debezium-source.version@-metadata.jar source.file=https://@repo-spring-io@/org/springframework/cloud/stream/app/file-source-kafka/@file-source.version@/file-source-kafka-@file-source.version@.jar source.file.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/file-source-kafka/@file-source.version@/file-source-kafka-@file-source.version@-metadata.jar source.ftp=https://@repo-spring-io@/org/springframework/cloud/stream/app/ftp-source-kafka/@ftp-source.version@/ftp-source-kafka-@ftp-source.version@.jar diff --git a/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/kafka-apps-maven.properties b/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/kafka-apps-maven.properties index 9980f80a..747b2498 100644 --- a/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/kafka-apps-maven.properties +++ b/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/kafka-apps-maven.properties @@ -1,3 +1,5 @@ +source.cdc-debezium=maven://org.springframework.cloud.stream.app:cdc-debezium-source-kafka:@cdc-debezium-source.version@ +source.cdc-debezium.metadata=maven://org.springframework.cloud.stream.app:cdc-debezium-source-kafka:jar:metadata:@cdc-debezium-source.version@ source.file=maven://org.springframework.cloud.stream.app:file-source-kafka:@file-source.version@ source.file.metadata=maven://org.springframework.cloud.stream.app:file-source-kafka:jar:metadata:@file-source.version@ source.ftp=maven://org.springframework.cloud.stream.app:ftp-source-kafka:@ftp-source.version@ diff --git a/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/rabbit-apps-docker.properties b/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/rabbit-apps-docker.properties index f4dd0d20..7e6dee99 100644 --- a/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/rabbit-apps-docker.properties +++ b/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/rabbit-apps-docker.properties @@ -1,3 +1,4 @@ +source.cdc-debezium=docker:springcloudstream/cdc-debezium-source-rabbit:@cdc-debezium-source-docker.tag@ source.file=docker:springcloudstream/file-source-rabbit:@file-source-docker.tag@ source.ftp=docker:springcloudstream/ftp-source-rabbit:@ftp-source-docker.tag@ source.geode=docker:springcloudstream/geode-source-rabbit:@geode-source-docker.tag@ diff --git a/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/rabbit-apps-maven-repo-url.properties b/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/rabbit-apps-maven-repo-url.properties index e09242d5..ecf76ea8 100644 --- a/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/rabbit-apps-maven-repo-url.properties +++ b/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/rabbit-apps-maven-repo-url.properties @@ -1,3 +1,5 @@ +source.cdc-debezium=https://@repo-spring-io@/org/springframework/cloud/stream/app/cdc-debezium-source-rabbit/@cdc-debezium-source.version@/cdc-debezium-source-rabbit-@cdc-debezium-source.version@.jar +source.cdc-debezium.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/cdc-debezium-source-rabbit/@cdc-debezium-source.version@/cdc-debezium-source-rabbit-@cdc-debezium-source.version@-metadata.jar source.file=https://@repo-spring-io@/org/springframework/cloud/stream/app/file-source-rabbit/@file-source.version@/file-source-rabbit-@file-source.version@.jar source.file.metadata=https://@repo-spring-io@/org/springframework/cloud/stream/app/file-source-rabbit/@file-source.version@/file-source-rabbit-@file-source.version@-metadata.jar source.ftp=https://@repo-spring-io@/org/springframework/cloud/stream/app/ftp-source-rabbit/@ftp-source.version@/ftp-source-rabbit-@ftp-source.version@.jar diff --git a/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/rabbit-apps-maven.properties b/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/rabbit-apps-maven.properties index f775e9d1..1a62a1ef 100644 --- a/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/rabbit-apps-maven.properties +++ b/applications/stream-applications-build/stream-applications-descriptor/src/main/resources/META-INF/rabbit-apps-maven.properties @@ -1,3 +1,5 @@ +source.cdc-debezium=maven://org.springframework.cloud.stream.app:cdc-debezium-source-rabbit:@cdc-debezium-source.version@ +source.cdc-debezium.metadata=maven://org.springframework.cloud.stream.app:cdc-debezium-source-rabbit:jar:metadata:@cdc-debezium-source.version@ source.file=maven://org.springframework.cloud.stream.app:file-source-rabbit:@file-source.version@ source.file.metadata=maven://org.springframework.cloud.stream.app:file-source-rabbit:jar:metadata:@file-source.version@ source.ftp=maven://org.springframework.cloud.stream.app:ftp-source-rabbit:@ftp-source.version@ diff --git a/applications/stream-applications-build/stream-applications-docs/pom.xml b/applications/stream-applications-build/stream-applications-docs/pom.xml index b2bce47f..0ff7e29d 100644 --- a/applications/stream-applications-build/stream-applications-docs/pom.xml +++ b/applications/stream-applications-build/stream-applications-docs/pom.xml @@ -26,6 +26,11 @@ Modules with @ConfigurationProperties. --> + + org.springframework.cloud.fn + cdc-debezium-common + ${java-functions.version} + org.springframework.cloud.fn file-common @@ -62,6 +67,11 @@ ${java-functions.version} + + org.springframework.cloud.fn + cdc-debezium-supplier + ${java-functions.version} + org.springframework.cloud.fn file-supplier diff --git a/applications/stream-applications-build/stream-applications-docs/src/main/asciidoc/sources.adoc b/applications/stream-applications-build/stream-applications-docs/src/main/asciidoc/sources.adoc index 5cdaa5b0..4f401599 100644 --- a/applications/stream-applications-build/stream-applications-docs/src/main/asciidoc/sources.adoc +++ b/applications/stream-applications-build/stream-applications-docs/src/main/asciidoc/sources.adoc @@ -3,6 +3,9 @@ :leveloffset: +2 +[[spring-cloud-stream-modules-cdc-debezium-source]] +include::{stream-apps-root}/{branch}/applications/source/cdc-debezium-source/README.adoc[tags=ref-doc] + [[spring-cloud-stream-modules-file-source]] include::{stream-apps-root}/{branch}/applications/source/file-source/README.adoc[tags=ref-doc] diff --git a/functions/common/cdc-debezium-boot-starter/README.adoc b/functions/common/cdc-debezium-boot-starter/README.adoc new file mode 100644 index 00000000..aa43bc12 --- /dev/null +++ b/functions/common/cdc-debezium-boot-starter/README.adoc @@ -0,0 +1,124 @@ += Debezium Spring Boot Starter + +Spring Boot Starter for easy integration of https://debezium.io[Debezium] in Spring Boot applications. + +To use it you just add the `cdc-debezium-boot-starter` dependency to your application POM and implement your own `Consumer` handler to process the incoming database change events. Follow the link below for further instructions. + +https://en.wikipedia.org/wiki/Change_data_capture[Change Data Capture] (CDC) Spring Boot Starter that allows capturing change events from various databases including `MySQL`, `PostgreSQL`, `MongoDB`, `Oracle` and `SQL Server`. + +== Quick Start + +https://start.spring.io/[Start new Spring Boot project] and add the following starter dependency to activate the CDC: + +[source, xml] +---- + + org.springframework.cloud.fn + cdc-debezium-boot-starter + 1.0.0-SNAPSHOT + +---- + +NOTE: for MySQL you need to adjust the mysql client version to `mysql:mysql-connector-java:8.0.13`. + +The CDC starter setups an embedded https://debezium.io/documentation/reference/0.10/connectors/index.html[debezium connector] to capture the database changes. + +Then implement a `Consumer` bean handler to receive and process the CDC notifications: + +[source, java] +---- +@Bean +public Consumer mySourceRecordConsumer( # <1> + Function keySerializer, # <2> + Function valueSerializer) { # <3> + + return sourceRecord -> { + System.out.println(sourceRecord.topic() # <4> + + " : " + new String(keySerializer.apply(sourceRecord)) # <5> + + " : " + new String(valueSerializer.apply(sourceRecord))); # <6> + }; +} +---- + +<1> CDC event listener that is called on every data change. +<2> Optional keySerializer converter that allows to serialize the keys of the SourceRecord into a compact binary format. +<3> Optional valueSerializer converter (by default JsonConverter) that allows to serialize the SourceRecord into a compact binary format. +<4> The `topic()` provides a logical identifier of the changed data (e.g. cdc-name.DB-name.table-name) +<5> Convert kay to text message using the keySerializer +<6> Convert payload to JSON message using the valueSerializer. + +NOTE: you may have to add `@AutoConfigureBefore(CdcAutoConfiguration.class)` to your Boot class to ensure that your record consumer overrides the default handlers. + +Above handler will produce output messages like this: +[source, bash] +---- +my-app-connector.inventory.addresses : {"id":10,"customer_id":1001,"street":"3183 Moore Avenue","city":"Euless","state":"Texas","zip":"76036","type":"SHIPPING"} +my-app-connector.inventory.customers : {"id":1004,"first_name":"Anne","last_name":"Kretchmar","email":"annek@noanswer.org"} +my-app-connector.inventory.orders : {"order_number":10001,"order_date":16816,"purchaser":1001,"quantity":1,"product_id":102} +my-app-connector.inventory.products : {"id":109,"name":"spare tire","description":"24 inch spare tire","weight":22.200000762939453} +my-app-connector.inventory.products_on_hand : {"product_id":101,"quantity":3} +---- + +All Debezium connector properties are supported. Just prefix those properties with `cdc.config` prefix. + +Following snipped shows how to configure the https://debezium.io/docs/connectors/mysql/[MySqlConnector] to connect to a local MySQL database run via the `debezium/example-mysql` docker image: +[source, bash] +---- +docker run -it --rm --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=debezium -e MYSQL_USER=mysqluser -e MYSQL_PASSWORD=mysqlpw debezium/example-mysql:1.0 +---- + +[TIP] +==== +(optional) Use `mysql` client to connected to the database and to create a `debezium` user with required credentials: +[source, bash] +---- +docker run -it --rm --name mysqlterm --link mysql --rm mysql:5.7 sh -c 'exec mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD"' +mysql> GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'debezium' IDENTIFIED BY 'dbz'; +---- +==== + +Configure `cdc-debezium-boot-starter` for consuming cdc events from the MySQL: + +[source] +---- +cdc.name=my-sql-connector # <1> + +cdc.connector=mysql # <2> + +cdc.config.database.server.id=85744 # <3> +cdc.config.database.server.name=my-app-connector # <3> +cdc.config.database.user=debezium # <3> +cdc.config.database.password=dbz # <3> +cdc.config.database.hostname=localhost # <3> +cdc.config.database.port=3306 # <3> + +cdc.schema=false # <4> + +cdc.flattering.enabled=true # <5> +---- + +<1> Metadata used to identify and dispatch the events received by this cdc consumer instance. +<2> Configures the CDC Source to use https://debezium.io/docs/connectors/mysql/[MySqlConnector]. (equivalent to setting `cdc.config.connector.class=io.debezium.connector.mysql.MySqlConnector`). +<3> Connector specific configurations. MySQL server logical name, connect location and access credentials. +<4> Do not serialize the record's schema in the output messages. +<5> Enables the https://debezium.io/docs/configuration/event-flattening/[CDC Event Flattering] feature. + +The full list of properties: + +==== Options + +//tag::configuration-properties[] +$$cdc.config$$:: $$Spring pass-trough wrapper for debezium configuration properties. All properties with a 'cdc.config.' prefix are native Debezium properties. The prefix is removed, converting them into Debezium io.debezium.config.Configuration.$$ *($$Map$$, default: `$$$$`)* +$$cdc.connector$$:: $$Shortcut for the cdc.config.connector.class property. Either of those can be used as long as they do not contradict with each other.$$ *($$ConnectorType$$, default: `$$$$`, possible values: `mysql`,`postgres`,`mongodb`,`oracle`,`sqlserver`)* +$$cdc.flattering.add-fields$$:: $$Comma separated list of metadata fields to add to the flattened message. The fields will be prefixed with "__" or "__[<]struct]__", depending on the specification of the struct.$$ *($$String$$, default: `$$$$`)* +$$cdc.flattering.add-headers$$:: $$Comma separated list specify a list of metadata fields to add to the header of the flattened message. The fields will be prefixed with "__" or "__[struct]__".$$ *($$String$$, default: `$$$$`)* +$$cdc.flattering.delete-handling-mode$$:: $$Options for handling deleted records: (1) none - pass the records through, (2) drop - remove the records and (3) rewrite - add a '__deleted' field to the records.$$ *($$DeleteHandlingMode$$, default: `$$$$`, possible values: `drop`,`rewrite`,`none`)* +$$cdc.flattering.drop-tombstones$$:: $$By default Debezium generates tombstone records to enable Kafka compaction on deleted records. The dropTombstones can suppress the tombstone records.$$ *($$Boolean$$, default: `$$true$$`)* +$$cdc.flattering.enabled$$:: $$Enable flattering the source record events (https://debezium.io/docs/configuration/event-flattening).$$ *($$Boolean$$, default: `$$true$$`)* +$$cdc.name$$:: $$Unique name for this sourceConnector instance.$$ *($$String$$, default: `$$$$`)* +$$cdc.offset.commit-timeout$$:: $$Maximum number of milliseconds to wait for records to flush and partition offset data to be committed to offset storage before cancelling the process and restoring the offset data to be committed in a future attempt.$$ *($$Duration$$, default: `$$5000ms$$`)* +$$cdc.offset.flush-interval$$:: $$Interval at which to try committing offsets. The default is 1 minute.$$ *($$Duration$$, default: `$$60000ms$$`)* +$$cdc.offset.policy$$:: $$Offset storage commit policy.$$ *($$OffsetPolicy$$, default: `$$$$`)* +$$cdc.offset.storage$$:: $$Kafka connector tracks the number processed records and regularly stores the count (as "offsets") in a preconfigured metadata storage. On restart the connector resumes the reading from the last recorded source offset.$$ *($$OffsetStorageType$$, default: `$$$$`, possible values: `memory`,`file`,`kafka`,`metadata`)* +$$cdc.schema$$:: $$Include the schema's as part of the outbound message.$$ *($$Boolean$$, default: `$$false$$`)* +//end::configuration-properties[] diff --git a/functions/common/cdc-debezium-boot-starter/pom.xml b/functions/common/cdc-debezium-boot-starter/pom.xml new file mode 100644 index 00000000..caa6310e --- /dev/null +++ b/functions/common/cdc-debezium-boot-starter/pom.xml @@ -0,0 +1,175 @@ + + + 4.0.0 + cdc-debezium-boot-starter + 1.0.0-SNAPSHOT + cdc-debezium-boot-starter + Change Data Capture (CDC) Debezium Boot Starter + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0-SNAPSHOT + ../../spring-functions-parent + + + + 1.2.1.Final + + + + + + org.springframework.cloud.fn + cdc-debezium-common + ${project.version} + + + slf4j-log4j12 + org.slf4j + + + + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + org.springframework.boot + spring-boot-starter-test + test + + + + + io.debezium + debezium-connector-mysql + + + mysql-connector-java + mysql + + + ${version.debezium} + + + + + + + + + + + + + + + + io.debezium + debezium-connector-postgres + + + slf4j-log4j12 + org.slf4j + + + ${version.debezium} + + + io.debezium + debezium-connector-oracle + + + slf4j-log4j12 + org.slf4j + + + ${version.debezium} + + + io.debezium + debezium-connector-sqlserver + + + slf4j-log4j12 + org.slf4j + + + ${version.debezium} + + + + mysql + mysql-connector-java + 8.0.13 + + + + org.springframework + spring-jdbc + test + + + + + + + + io.fabric8 + docker-maven-plugin + 0.33.0 + + + + mysql + debezium/example-mysql:1.0 + + + debezium + mysqluser + mysqlpw + + + 3306:3306 + + + port: 3306 + + + + + + + + + start + + generate-test-resources + + build + start + + + + stop + post-integration-test + + stop + + + + stop-pre + clean + + stop + + + + + + + diff --git a/functions/common/cdc-debezium-boot-starter/src/main/java/org/springframework/cloud/fn/common/cdc/CdcAutoConfiguration.java b/functions/common/cdc-debezium-boot-starter/src/main/java/org/springframework/cloud/fn/common/cdc/CdcAutoConfiguration.java new file mode 100644 index 00000000..e073f0cc --- /dev/null +++ b/functions/common/cdc-debezium-boot-starter/src/main/java/org/springframework/cloud/fn/common/cdc/CdcAutoConfiguration.java @@ -0,0 +1,71 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.common.cdc; + +import java.util.function.Consumer; +import java.util.function.Function; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kafka.connect.source.SourceRecord; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +/** + * @author Christian Tzolov + */ +@Configuration +@Import(CdcCommonConfiguration.class) +public class CdcAutoConfiguration { + + private static final Log logger = LogFactory.getLog(CdcAutoConfiguration.class); + + @Bean + @ConditionalOnMissingBean + public Consumer defaultSourceRecordConsumer() { + return sourceRecord -> logger.info("[CDC Event]: " + ((sourceRecord == null) ? "null" : sourceRecord.toString())); + } + + @Bean + public EmbeddedEngineExecutorService embeddedEngine(EmbeddedEngine.Builder embeddedEngineBuilder, + Consumer sourceRecordConsumer, Function recordFlattering) { + + EmbeddedEngine embeddedEngine = embeddedEngineBuilder + .notifying(sourceRecord -> sourceRecordConsumer.accept(recordFlattering.apply(sourceRecord))) + .build(); + + return new EmbeddedEngineExecutorService(embeddedEngine) { + @PostConstruct + @Override + public void start() { + super.start(); + } + + @PreDestroy + @Override + public void close() { + super.close(); + } + }; + } +} diff --git a/functions/common/cdc-debezium-boot-starter/src/main/resources/META-INF/spring.factories b/functions/common/cdc-debezium-boot-starter/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000..cc6cecc9 --- /dev/null +++ b/functions/common/cdc-debezium-boot-starter/src/main/resources/META-INF/spring.factories @@ -0,0 +1 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=org.springframework.cloud.fn.common.cdc.CdcAutoConfiguration diff --git a/functions/common/cdc-debezium-boot-starter/src/test/java/org/springframework/cloud/fn/common/cdc/CdcBootStarterIntegrationTest.java b/functions/common/cdc-debezium-boot-starter/src/test/java/org/springframework/cloud/fn/common/cdc/CdcBootStarterIntegrationTest.java new file mode 100644 index 00000000..f9d92e10 --- /dev/null +++ b/functions/common/cdc-debezium-boot-starter/src/test/java/org/springframework/cloud/fn/common/cdc/CdcBootStarterIntegrationTest.java @@ -0,0 +1,80 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.common.cdc; + +import org.junit.Test; + +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.springframework.test.jdbc.JdbcTestUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Christian Tzolov + */ +public class CdcBootStarterIntegrationTest { + + private final JdbcTemplate jdbcTemplate = jdbcTemplate( + "com.mysql.cj.jdbc.Driver", + "jdbc:mysql://localhost:3306/inventory", + "root", + "debezium"); + + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withUserConfiguration(TestCdcApplication.class) + .withPropertyValues( + "cdc.name=my-sql-connector", + "cdc.schema=false", + "cdc.flattering.enabled=true", + "cdc.stream.header.offset=true", + "cdc.connector=mysql", + "cdc.config.database.user=debezium", + "cdc.config.database.password=dbz", + "cdc.config.database.hostname=localhost", + "cdc.config.database.port=3306", + "cdc.config.database.server.id=85744", + "cdc.config.database.server.name=my-app-connector", + "cdc.config.database.history=io.debezium.relational.history.MemoryDatabaseHistory"); + + @Test + public void consumerTest() { + contextRunner + .withPropertyValues( + "cdc.flattering.deleteHandlingMode=drop", + "cdc.flattering.dropTombstones=true") + .run(context -> { + TestCdcApplication.TestSourceRecordConsumer testConsumer = + context.getBean(TestCdcApplication.TestSourceRecordConsumer.class); + jdbcTemplate.update("insert into `customers`(`first_name`,`last_name`,`email`) VALUES('Test666', 'Test666', 'Test666@spring.org')"); + JdbcTestUtils.deleteFromTableWhere(jdbcTemplate, "customers", "first_name = ?", "Test666"); + + Thread.sleep(10000); + assertThat(testConsumer.recordList).hasSize(53); + }); + } + + public static JdbcTemplate jdbcTemplate(String jdbcDriver, String jdbcUrl, String user, String password) { + DriverManagerDataSource dataSource = new DriverManagerDataSource(); + dataSource.setDriverClassName(jdbcDriver); + dataSource.setUrl(jdbcUrl); + dataSource.setUsername(user); + dataSource.setPassword(password); + return new JdbcTemplate(dataSource); + } +} diff --git a/functions/common/cdc-debezium-boot-starter/src/test/java/org/springframework/cloud/fn/common/cdc/TestCdcApplication.java b/functions/common/cdc-debezium-boot-starter/src/test/java/org/springframework/cloud/fn/common/cdc/TestCdcApplication.java new file mode 100644 index 00000000..bf12cd31 --- /dev/null +++ b/functions/common/cdc-debezium-boot-starter/src/test/java/org/springframework/cloud/fn/common/cdc/TestCdcApplication.java @@ -0,0 +1,79 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.common.cdc; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; + +import org.apache.kafka.connect.source.SourceRecord; + +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.integration.metadata.SimpleMetadataStore; + +/** + * @author Christian Tzolov + */ +@SpringBootConfiguration +@EnableAutoConfiguration +public class TestCdcApplication { + + @Bean + public Consumer mySourceRecordConsumer(Function valueSerializer, + Function keySerializer) { + return new TestSourceRecordConsumer(valueSerializer, keySerializer); + } + + @Bean + public SimpleMetadataStore simpleMetadataStore() { + return new SimpleMetadataStore(); + } + + public static class TestSourceRecordConsumer implements Consumer { + + private final Function valueSerializer; + private final Function keySerializer; + + public Map keyValue = new HashMap<>(); + public List recordList = new ArrayList<>(); + + public TestSourceRecordConsumer(Function valueSerializer, + Function keySerializer) { + this.valueSerializer = valueSerializer; + this.keySerializer = keySerializer; + } + + @Override + public void accept(SourceRecord sourceRecord) { + if (sourceRecord != null) { // ignore null records + recordList.add(sourceRecord); + Object payload = valueSerializer.apply(sourceRecord); + Object key = keySerializer.apply(sourceRecord); + keyValue.put(key, payload); + + System.out.println("[CDC Event]: " + sourceRecord.toString()); + } + } + } +} + + diff --git a/functions/common/cdc-debezium-common/pom.xml b/functions/common/cdc-debezium-common/pom.xml new file mode 100644 index 00000000..e6d37898 --- /dev/null +++ b/functions/common/cdc-debezium-common/pom.xml @@ -0,0 +1,117 @@ + + + 4.0.0 + cdc-debezium-common + 1.0.0-SNAPSHOT + cdc-debezium-common + Change Data Capture (CDC) Debezium Common + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0-SNAPSHOT + ../../spring-functions-parent + + + + 1.2.1.Final + + 42.2.5 + 5.2.1.RELEASE + + 5.1.46 + + + + + io.debezium + debezium-embedded + ${version.debezium} + + + + io.debezium + debezium-connector-mysql + + + mysql-connector-java + mysql + + + true + ${version.debezium} + + + + mysql + mysql-connector-java + true + ${mysql-connector-java} + + + + io.debezium + debezium-connector-postgres + true + ${version.debezium} + + + io.debezium + debezium-connector-mongodb + true + ${version.debezium} + + + io.debezium + debezium-connector-oracle + true + ${version.debezium} + + + io.debezium + debezium-connector-sqlserver + true + ${version.debezium} + + + + org.springframework.cloud.fn + metadata-store-common + ${project.version} + + + + + org.springframework.boot + spring-boot-starter-json + + + org.springframework.boot + spring-boot-starter-validation + + + + org.springframework.integration + spring-integration-ip + + + org.springframework.cloud.fn + config-common + ${spring-cloud-fn.version} + + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + org.springframework.boot + spring-boot-starter-test + test + + + + + diff --git a/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/CdcCommonConfiguration.java b/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/CdcCommonConfiguration.java new file mode 100644 index 00000000..434b8e7a --- /dev/null +++ b/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/CdcCommonConfiguration.java @@ -0,0 +1,131 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.common.cdc; + +import java.util.Collections; +import java.util.Map; +import java.util.function.Function; + +import io.debezium.transforms.ExtractNewRecordState; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kafka.connect.json.JsonConverter; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.storage.Converter; +import org.apache.kafka.connect.storage.OffsetBackingStore; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.util.StringUtils; + +/** + * @author Christian Tzolov + */ +@Configuration +@EnableConfigurationProperties(CdcCommonProperties.class) +@Import(CdcOffsetBackingStoreConfiguration.class) +public class CdcCommonConfiguration { + + private static final Log logger = LogFactory.getLog(CdcCommonConfiguration.class); + + @Bean + public io.debezium.config.Configuration configuration(CdcCommonProperties properties) { + Map configMap = properties.getConfig(); + return io.debezium.config.Configuration.from(configMap); + } + + @Bean + public Function recordFlattering(CdcCommonProperties properties, + ExtractNewRecordState extractNewRecordState) { + return sourceRecord -> properties.getFlattering().isEnabled() ? + (SourceRecord) extractNewRecordState.apply(sourceRecord) : sourceRecord; + } + + @Bean + public ExtractNewRecordState extractNewRecordState(CdcCommonProperties properties) { + ExtractNewRecordState extractNewRecordState = new ExtractNewRecordState(); + Map config = extractNewRecordState.config().defaultValues(); + config.put("drop.tombstones", properties.getFlattering().isDropTombstones()); + config.put("delete.handling.mode", properties.getFlattering().getDeleteHandlingMode().name()); + if (!StringUtils.isEmpty(properties.getFlattering().getAddHeaders())) { + config.put("add.headers", properties.getFlattering().getAddHeaders()); + } + if (!StringUtils.isEmpty(properties.getFlattering().getAddFields())) { + config.put("add.fields", properties.getFlattering().getAddFields()); + } + + extractNewRecordState.configure(config); + return extractNewRecordState; + } + + @Bean + @ConditionalOnMissingBean + public JsonConverter jsonConverter(CdcCommonProperties properties) { + JsonConverter jsonConverter = new JsonConverter(); + jsonConverter.configure(Collections.singletonMap("schemas.enable", properties.isSchema()), false); + return jsonConverter; + } + + @Bean + public Function valueSerializer(Converter valueConverter) { + return sourceRecord -> valueConverter.fromConnectData( + sourceRecord.topic(), sourceRecord.valueSchema(), sourceRecord.value()); + } + + @Bean + public Function keySerializer(Converter valueConverter) { + return sourceRecord -> valueConverter.fromConnectData( + sourceRecord.topic(), sourceRecord.keySchema(), sourceRecord.key()); + } + + @Bean + public EmbeddedEngine.Builder embeddedEngineBuilder(CdcCommonProperties properties, + OffsetBackingStore offsetBackingStore) { + + if (!properties.getConfig().containsKey("connector.class")) { + properties.getConfig().put("connector.class", properties.getConnector().connectorClass); + } + + if (!properties.getConfig().containsKey("name")) { + properties.getConfig().put("name", properties.getName()); + } + + if (!properties.getConfig().containsKey("offset.flush.interval.ms")) { + properties.getConfig().put("offset.flush.interval.ms", properties.getOffset().getFlushInterval().toMillis() + ""); + } + + if (!properties.getConfig().containsKey("offset.flush.timeout.ms")) { + properties.getConfig().put("offset.flush.timeout.ms", properties.getOffset().getCommitTimeout().toMillis() + ""); + } + + if (!properties.getConfig().containsKey("offset.commit.policy")) { + properties.getConfig().put("offset.commit.policy", properties.getOffset().getPolicy().policyClass); + } + + if (!properties.getConfig().containsKey("offset.storage")) { + properties.getConfig().put("offset.storage", properties.getOffset().getStorage().offsetStorageClass); + } + + return EmbeddedEngine.create() + .using(io.debezium.config.Configuration.from(properties.getConfig())) + .offsetBackingStore(offsetBackingStore); + } + +} diff --git a/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/CdcCommonProperties.java b/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/CdcCommonProperties.java new file mode 100644 index 00000000..7756c75b --- /dev/null +++ b/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/CdcCommonProperties.java @@ -0,0 +1,312 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.common.cdc; + +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; + +import javax.validation.constraints.AssertTrue; +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +/** + * + * @author Christian Tzolov + */ +@ConfigurationProperties("cdc") +@Validated +public class CdcCommonProperties { + + /** + * Unique name for this sourceConnector instance. + */ + @NotEmpty + private String name; + + /** + * Shortcut for the cdc.config.connector.class property. Either of those can be used as long as they do not + * contradict with each other. + */ + @NotNull + private ConnectorType connector = null; + + private final Offset offset = new Offset(); + + /** + * Include the schema's as part of the outbound message. + */ + private boolean schema = false; + + /** + * Event Flattering (https://debezium.io/docs/configuration/event-flattening). + */ + private final Flattering flattering = new Flattering(); + + /** + * Spring pass-trough wrapper for debezium configuration properties. + * All properties with a 'cdc.config.' prefix are native Debezium properties. + * The prefix is removed, converting them into Debezium io.debezium.config.Configuration. + */ + private Map config = defaultConfig(); + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Offset getOffset() { + return offset; + } + + public Flattering getFlattering() { + return flattering; + } + + public Map getConfig() { + return config; + } + + public boolean isSchema() { + return schema; + } + + public void setSchema(boolean schema) { + this.schema = schema; + } + + private Map defaultConfig() { + Map defaultConfig = new HashMap<>(); + defaultConfig.put("database.history", "io.debezium.relational.history.MemoryDatabaseHistory"); + //defaultConfig.put("offset.flush.interval.ms", "60000"); + return defaultConfig; + } + + public ConnectorType getConnector() { + return connector; + } + + public void setConnector(ConnectorType connector) { + this.connector = connector; + } + + @AssertTrue + public boolean connectorIsSet() { + return this.getConnector() != null || this.getConfig().containsKey("connector.class"); + } + + public static class Offset { + + /** + * Interval at which to try committing offsets. The default is 1 minute. + */ + private Duration flushInterval = Duration.ofMillis(60000); + + /** + * Maximum number of milliseconds to wait for records to flush and partition offset data to be committed to + * offset storage before cancelling the process and restoring the offset data to be committed in a future attempt. + */ + private Duration commitTimeout = Duration.ofMillis(5000); + + /** + * Offset storage commit policy. + */ + private OffsetPolicy policy = OffsetPolicy.periodic; + + /** + * Kafka connector tracks the number processed records and regularly stores the count (as "offsets") in a + * preconfigured metadata storage. On restart the connector resumes the reading from the last recorded source offset. + */ + private OffsetStorageType storage = OffsetStorageType.metadata; + + public enum OffsetPolicy { + /** periodic. */ + periodic("io.debezium.embedded.spi.OffsetCommitPolicy$PeriodicCommitOffsetPolicy"), + /** always. */ + always("OffsetCommitPolicy.AlwaysCommitOffsetPolicy.class.getName()"); + + /** policy. */ + public final String policyClass; + + OffsetPolicy(String policyClassName) { + this.policyClass = policyClassName; + } + } + + public Duration getFlushInterval() { + return flushInterval; + } + + public void setFlushInterval(Duration flushInterval) { + this.flushInterval = flushInterval; + } + + public Duration getCommitTimeout() { + return commitTimeout; + } + + public void setCommitTimeout(Duration commitTimeout) { + this.commitTimeout = commitTimeout; + } + + public OffsetPolicy getPolicy() { + return policy; + } + + public void setPolicy(OffsetPolicy policy) { + this.policy = policy; + } + + public OffsetStorageType getStorage() { + return storage; + } + + public void setStorage(OffsetStorageType storage) { + this.storage = storage; + } + + } + + public enum OffsetStorageType { + /** memory offset storage type. */ + memory("org.apache.kafka.connect.storage.MemoryOffsetBackingStore"), + /** File offset storage type. */ + file("org.apache.kafka.connect.storage.FileOffsetBackingStore"), + /** Kafka offset storage type. */ + kafka("org.apache.kafka.connect.storage.KafkaOffsetBackingStore"), + /** Metadata offset storage type. */ + metadata("org.springframework.cloud.stream.app.cdc.common.core.store.MetadataStoreOffsetBackingStore"); + + /** Class name of the Offset Storage. */ + public final String offsetStorageClass; + + OffsetStorageType(String type) { + this.offsetStorageClass = type; + } + } + + public enum ConnectorType { + /** MySql connector type. */ + mysql("io.debezium.connector.mysql.MySqlConnector"), + /** Postgres connector type. */ + postgres("io.debezium.connector.postgresql.PostgresConnector"), + /** Mongodb connector type. */ + mongodb("io.debezium.connector.mongodb.MongoDbConnector"), + /** Oracle connector type. */ + oracle("io.debezium.connector.oracle.OracleConnector"), + /** SqlServer connector type. */ + sqlserver("io.debezium.connector.sqlserver.SqlServerConnector"); + + /** Connector class name. */ + public final String connectorClass; + + ConnectorType(String type) { + this.connectorClass = type; + } + } + + public enum DeleteHandlingMode { + /** records are removed. */ + drop, + /** add a __deleted column with true/false values based on record operation. */ + rewrite, + /** pass delete events. */ + none + } + + /** + * https://debezium.io/documentation/reference/0.10/configuration/event-flattening.html . + * https://debezium.io/documentation/reference/0.10/configuration/event-flattening.html#configuration_options + */ + public static class Flattering { + + /** + * Enable flattering the source record events (https://debezium.io/docs/configuration/event-flattening). + */ + private boolean enabled = true; + + /** + * By default Debezium generates tombstone records to enable Kafka compaction on deleted records. + * The dropTombstones can suppress the tombstone records. + */ + private boolean dropTombstones = true; + + /** + * Options for handling deleted records: (1) none - pass the records through, (2) drop - remove the records and + * (3) rewrite - add a '__deleted' field to the records. + */ + private DeleteHandlingMode deleteHandlingMode = DeleteHandlingMode.drop; + + /** + * Comma separated list of metadata fields to add to the flattened message. + * The fields will be prefixed with "__" or "__[<]struct]__", depending on the specification of the struct. + */ + private String addFields = null; + + /** + * Comma separated list specify a list of metadata fields to add to the header of the flattened message. + * The fields will be prefixed with "__" or "__[struct]__". + */ + private String addHeaders = null; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public boolean isDropTombstones() { + return dropTombstones; + } + + public void setDropTombstones(boolean dropTombstones) { + this.dropTombstones = dropTombstones; + } + + public DeleteHandlingMode getDeleteHandlingMode() { + return deleteHandlingMode; + } + + public void setDeleteHandlingMode(DeleteHandlingMode deleteHandlingMode) { + this.deleteHandlingMode = deleteHandlingMode; + } + + public String getAddFields() { + return addFields; + } + + public void setAddFields(String addFields) { + this.addFields = addFields; + } + + public String getAddHeaders() { + return addHeaders; + } + + public void setAddHeaders(String addHeaders) { + this.addHeaders = addHeaders; + } + } +} diff --git a/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/CdcOffsetBackingStoreConfiguration.java b/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/CdcOffsetBackingStoreConfiguration.java new file mode 100644 index 00000000..ce677211 --- /dev/null +++ b/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/CdcOffsetBackingStoreConfiguration.java @@ -0,0 +1,68 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.common.cdc; + +import org.apache.kafka.connect.storage.FileOffsetBackingStore; +import org.apache.kafka.connect.storage.KafkaOffsetBackingStore; +import org.apache.kafka.connect.storage.MemoryOffsetBackingStore; +import org.apache.kafka.connect.storage.OffsetBackingStore; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.cloud.fn.common.cdc.store.MetadataStoreOffsetBackingStore; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.integration.metadata.MetadataStore; + +/** + * @author Christian Tzolov + */ +@Configuration +public class CdcOffsetBackingStoreConfiguration { + + @Bean + @ConditionalOnMissingBean + @ConditionalOnExpression("'${cdc.config.offset.storage}'.equalsIgnoreCase('org.springframework.cloud.stream.app.cdc.common.core.store.MetadataStoreOffsetBackingStore') " + + "or '${cdc.offset.storage:metadata}'.equals('metadata')") + public OffsetBackingStore metadataStoreOffsetBackingStore(MetadataStore metadataStore) { + return new MetadataStoreOffsetBackingStore(metadataStore); + } + + @Bean + @ConditionalOnMissingBean + @ConditionalOnExpression("'${cdc.config.offset.storage}'.equalsIgnoreCase('org.apache.kafka.connect.storage.FileOffsetBackingStore') " + + "or '${cdc.offset.storage:metadata}'.equalsIgnoreCase('file')") + public OffsetBackingStore fileOffsetBackingStore() { + return new FileOffsetBackingStore(); + } + + @Bean + @ConditionalOnMissingBean + @ConditionalOnExpression("'${cdc.config.offset.storage}'.equalsIgnoreCase('org.apache.kafka.connect.storage.KafkaOffsetBackingStore') " + + "or '${cdc.offset.storage:metadata}'.equalsIgnoreCase('kafka')") + public OffsetBackingStore kafkaOffsetBackingStore() { + return new KafkaOffsetBackingStore(); + } + + @Bean + @ConditionalOnMissingBean + @ConditionalOnExpression("'${cdc.config.offset.storage}'.equalsIgnoreCase('org.apache.kafka.connect.storage.MemoryOffsetBackingStore') " + + "or '${cdc.offset.storage:metadata}'.equalsIgnoreCase('memory')") + public OffsetBackingStore memoryOffsetBackingStore() { + return new MemoryOffsetBackingStore(); + } +} diff --git a/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/EmbeddedEngine.java b/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/EmbeddedEngine.java new file mode 100644 index 00000000..03fec9f9 --- /dev/null +++ b/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/EmbeddedEngine.java @@ -0,0 +1,1074 @@ +/* + * Copyright Debezium Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package org.springframework.cloud.fn.common.cdc; + +import java.io.IOException; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import io.debezium.annotation.ThreadSafe; +import io.debezium.config.Configuration; +import io.debezium.config.Field; +import io.debezium.config.Instantiator; +import io.debezium.embedded.StopConnectorException; +import io.debezium.embedded.Transformations; +import io.debezium.engine.DebeziumEngine; +import io.debezium.engine.StopEngineException; +import io.debezium.engine.spi.OffsetCommitPolicy; +import io.debezium.pipeline.ChangeEventSourceCoordinator; +import io.debezium.util.Clock; +import io.debezium.util.VariableLatch; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigDef.Importance; +import org.apache.kafka.common.config.ConfigDef.Type; +import org.apache.kafka.common.config.ConfigDef.Width; +import org.apache.kafka.connect.connector.ConnectorContext; +import org.apache.kafka.connect.connector.Task; +import org.apache.kafka.connect.errors.RetriableException; +import org.apache.kafka.connect.json.JsonConverter; +import org.apache.kafka.connect.runtime.WorkerConfig; +import org.apache.kafka.connect.runtime.distributed.DistributedConfig; +import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; +import org.apache.kafka.connect.source.SourceConnector; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.source.SourceTask; +import org.apache.kafka.connect.source.SourceTaskContext; +import org.apache.kafka.connect.storage.Converter; +import org.apache.kafka.connect.storage.FileOffsetBackingStore; +import org.apache.kafka.connect.storage.KafkaOffsetBackingStore; +import org.apache.kafka.connect.storage.OffsetBackingStore; +import org.apache.kafka.connect.storage.OffsetStorageReader; +import org.apache.kafka.connect.storage.OffsetStorageReaderImpl; +import org.apache.kafka.connect.storage.OffsetStorageWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A mechanism for running a single Kafka Connect {@link SourceConnector} within an application's process. An embedded connector + * is entirely standalone and only talks with the source system; no Kafka, Kafka Connect, or Zookeeper processes are needed. + * Applications using an embedded connector simply set one up and supply a {@link Consumer consumer function} to which the + * connector will pass all {@link SourceRecord}s containing database change events. + *

+ * With an embedded connector, the application that runs the connector assumes all responsibility for fault tolerance, + * scalability, and durability. Additionally, applications must specify how the connector can store its relational database + * schema history and offsets. By default, this information will be stored in memory and will thus be lost upon application + * restart. + *

+ * Embedded connectors are designed to be submitted to an {@link Executor} or {@link ExecutorService} for execution by a single + * thread, and a running connector can be stopped either by calling {@link #stop()} from another thread or by interrupting + * the running thread (e.g., as is the case with {@link ExecutorService#shutdownNow()}). + * + * @author Randall Hauch + */ +@ThreadSafe +public final class EmbeddedEngine implements DebeziumEngine { + + /** + * A required field for an embedded connector that specifies the unique name for the connector instance. + */ + public static final Field ENGINE_NAME = Field.create("name") + .withDescription("Unique name for this connector instance.") + .withValidation(Field::isRequired); + + /** + * A required field for an embedded connector that specifies the name of the normal Debezium connector's Java class. + */ + public static final Field CONNECTOR_CLASS = Field.create("connector.class") + .withDescription("The Java class for the connector") + .withValidation(Field::isRequired); + + /** + * An optional field that specifies the name of the class that implements the {@link OffsetBackingStore} interface, + * and that will be used to store offsets recorded by the connector. + */ + public static final Field OFFSET_STORAGE = Field.create("offset.storage") + .withDescription("The Java class that implements the `OffsetBackingStore` " + + "interface, used to periodically store offsets so that, upon " + + "restart, the connector can resume where it last left off.") + .withDefault(FileOffsetBackingStore.class.getName()); + + /** + * An optional field that specifies the file location for the {@link FileOffsetBackingStore}. + * + * @see #OFFSET_STORAGE + */ + public static final Field OFFSET_STORAGE_FILE_FILENAME = Field.create(StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG) + .withDescription("The file where offsets are to be stored. Required when " + + "'offset.storage' is set to the " + + FileOffsetBackingStore.class.getName() + " class.") + .withDefault(""); + + /** + * An optional field that specifies the topic name for the {@link KafkaOffsetBackingStore}. + * + * @see #OFFSET_STORAGE + */ + public static final Field OFFSET_STORAGE_KAFKA_TOPIC = Field.create(DistributedConfig.OFFSET_STORAGE_TOPIC_CONFIG) + .withDescription("The name of the Kafka topic where offsets are to be stored. " + + "Required with other properties when 'offset.storage' is set to the " + + KafkaOffsetBackingStore.class.getName() + " class.") + .withDefault(""); + + /** + * An optional field that specifies the number of partitions for the {@link KafkaOffsetBackingStore}. + * + * @see #OFFSET_STORAGE + */ + public static final Field OFFSET_STORAGE_KAFKA_PARTITIONS = Field.create(DistributedConfig.OFFSET_STORAGE_PARTITIONS_CONFIG) + .withType(ConfigDef.Type.INT) + .withDescription("The number of partitions used when creating the offset storage topic. " + + "Required with other properties when 'offset.storage' is set to the " + + KafkaOffsetBackingStore.class.getName() + " class."); + + /** + * An optional field that specifies the replication factor for the {@link KafkaOffsetBackingStore}. + * + * @see #OFFSET_STORAGE + */ + public static final Field OFFSET_STORAGE_KAFKA_REPLICATION_FACTOR = Field.create(DistributedConfig.OFFSET_STORAGE_REPLICATION_FACTOR_CONFIG) + .withType(ConfigDef.Type.SHORT) + .withDescription("Replication factor used when creating the offset storage topic. " + + "Required with other properties when 'offset.storage' is set to the " + + KafkaOffsetBackingStore.class.getName() + " class."); + + /** + * An optional advanced field that specifies the maximum amount of time that the embedded connector should wait + * for an offset commit to complete. + */ + public static final Field OFFSET_FLUSH_INTERVAL_MS = Field.create("offset.flush.interval.ms") + .withDescription("Interval at which to try committing offsets. The default is 1 minute.") + .withDefault(60000L) + .withValidation(Field::isNonNegativeInteger); + + /** + * An optional advanced field that specifies the maximum amount of time that the embedded connector should wait + * for an offset commit to complete. + */ + public static final Field OFFSET_COMMIT_TIMEOUT_MS = Field.create("offset.flush.timeout.ms") + .withDescription("Maximum number of milliseconds to wait for records to flush and partition offset data to be" + + " committed to offset storage before cancelling the process and restoring the offset " + + "data to be committed in a future attempt.") + .withDefault(5000L) + .withValidation(Field::isPositiveInteger); + + public static final Field OFFSET_COMMIT_POLICY = Field.create("offset.commit.policy") + .withDescription("The fully-qualified class name of the commit policy type. This class must implement the interface " + + OffsetCommitPolicy.class.getName() + + ". The default is a periodic commit policy based upon time intervals.") + .withDefault(io.debezium.embedded.spi.OffsetCommitPolicy.PeriodicCommitOffsetPolicy.class.getName()) + .withValidation(Field::isClassName); + + protected static final Field INTERNAL_KEY_CONVERTER_CLASS = Field.create("internal.key.converter") + .withDescription("The Converter class that should be used to serialize and deserialize key data for offsets.") + .withDefault(JsonConverter.class.getName()); + + protected static final Field INTERNAL_VALUE_CONVERTER_CLASS = Field.create("internal.value.converter") + .withDescription("The Converter class that should be used to serialize and deserialize value data for offsets.") + .withDefault(JsonConverter.class.getName()); + + /** + * A list of SMTs to be applied on the messages generated by the engine. + */ + public static final Field TRANSFORMS = Field.create("transforms") + .withDisplayName("List of prefixes defining transformations.") + .withType(Type.STRING) + .withWidth(Width.MEDIUM) + .withImportance(Importance.LOW) + .withDescription("Optional list of single message transformations applied on the messages. " + + "The transforms are defined using '.type' config option and configured using options '.

+ * This method returns immediately if the connector has completed already. + * + * @throws InterruptedException if the current thread is interrupted while waiting + */ + public void await() throws InterruptedException { + this.completed.await(); + } + + /** + * Causes the current thread to wait until the {@link #handle(boolean, String, Throwable) completion occurs}, + * unless the thread is {@linkplain Thread#interrupt interrupted}, or the specified waiting time elapses. + *

+ * This method returns immediately if the connector has completed already. + * + * @param timeout the maximum time to wait + * @param unit the time unit of the {@code timeout} argument + * @return {@code true} if the completion was received, or {@code false} if the waiting time elapsed before the completion + * was received. + * @throws InterruptedException if the current thread is interrupted while waiting + */ + public boolean await(long timeout, TimeUnit unit) throws InterruptedException { + return this.completed.await(timeout, unit); + } + + /** + * Determine if the connector has completed. + * + * @return {@code true} if the connector has completed, or {@code false} if the connector is still running and this + * callback has not yet been {@link #handle(boolean, String, Throwable) notified} + */ + public boolean hasCompleted() { + return completed.getCount() == 0; + } + + /** + * Get whether the connector completed normally. + * + * @return {@code true} if the connector completed normally, or {@code false} if the connector produced an error that + * prevented startup or premature termination (or the connector has not yet {@link #hasCompleted() completed}) + */ + public boolean success() { + return success; + } + + /** + * Get the completion message. + * + * @return the completion message, or null if the connector has not yet {@link #hasCompleted() completed} + */ + public String message() { + return message; + } + + /** + * Get the completion error, if there is one. + * + * @return the completion error, or null if there is no error or connector has not yet {@link #hasCompleted() completed} + */ + public Throwable error() { + return error; + } + + /** + * Determine if there is a completion error. + * + * @return {@code true} if there is a {@link #error completion error}, or {@code false} if there is no error or + * the connector has not yet {@link #hasCompleted() completed} + */ + public boolean hasError() { + return error != null; + } + } + + /** + * Contract passed to {@link ChangeConsumer}s, allowing them to commit single records as they have been processed + * and to signal that offsets may be flushed eventually. + */ + @ThreadSafe + @Deprecated + public static interface RecordCommitter extends DebeziumEngine.RecordCommitter { + } + + /** + * A contract invoked by the embedded engine when it has received a batch of change records to be processed. Allows + * to process multiple records in one go, acknowledging their processing once that's done. + */ + @Deprecated + public static interface ChangeConsumer extends DebeziumEngine.ChangeConsumer { + } + + private static ChangeConsumer buildDefaultChangeConsumer(Consumer consumer) { + return new ChangeConsumer() { + + /** + * the default implementation that is compatible with the old Consumer api. + * + * On every record, it calls the consumer, and then only marks the record + * as processed when accept returns, additionally, it handles StopConnectorExceptions + * and ensures that we all ways try and mark a batch as finished, even with exceptions + * @param records the records to be processed + * @param committer the committer that indicates to the system that we are finished + * + * @throws Exception + */ + @Override + public void handleBatch(List records, DebeziumEngine.RecordCommitter committer) throws InterruptedException { + for (SourceRecord record : records) { + try { + consumer.accept(record); + committer.markProcessed(record); + } + catch (StopConnectorException | StopEngineException ex) { + // ensure that we mark the record as finished + // in this case + committer.markProcessed(record); + throw ex; + } + } + committer.markBatchFinished(); + } + }; + } + + /** + * A builder to set up and create {@link EmbeddedEngine} instances. + */ + @Deprecated + public static interface Builder extends DebeziumEngine.Builder { + + /** + * Use the specified configuration for the connector. The configuration is assumed to already be valid. + * + * @param config the configuration + * @return this builder object so methods can be chained together; never null + */ + Builder using(Configuration config); + + /** + * Use the specified clock when needing to determine the current time. Passing null or not calling this + * method results in the connector using the {@link Clock#system() system clock}. + * + * @param clock the clock + * @return this builder object so methods can be chained together; never null + */ + Builder using(Clock clock); + + // backward compatibility methods + @Override + Builder notifying(Consumer consumer); + + @Override + Builder notifying(DebeziumEngine.ChangeConsumer handler); + + @Override + Builder using(ClassLoader classLoader); + + Builder using(CompletionCallback completionCallback); + + Builder using(ConnectorCallback connectorCallback); + + @Override + Builder using(OffsetCommitPolicy policy); + + Builder sourceConnector(SourceConnector sourceConnector); + + Builder offsetBackingStore(OffsetBackingStore offsetBackingStore); + + @Override + EmbeddedEngine build(); + } + + /** + * Obtain a new {@link Builder} instance that can be used to construct runnable {@link EmbeddedEngine} instances. + * + * @return the new builder; never null + */ + @Deprecated + public static Builder create() { + return new BuilderImpl(); + } + + private static final Logger LOGGER = LoggerFactory.getLogger(EmbeddedEngine.class); + + private final Configuration config; + private final Clock clock; + private final ClassLoader classLoader; + private final DebeziumEngine.ChangeConsumer handler; + private final DebeziumEngine.CompletionCallback completionCallback; + private final DebeziumEngine.ConnectorCallback connectorCallback; + private final AtomicReference runningThread = new AtomicReference<>(); + private final VariableLatch latch = new VariableLatch(0); + private final Converter keyConverter; + private final Converter valueConverter; + private final WorkerConfig workerConfig; + private final CompletionResult completionResult; + private long recordsSinceLastCommit = 0; + private long timeOfLastCommitMillis = 0; + private OffsetCommitPolicy offsetCommitPolicy; + private SourceConnector connector; + private OffsetBackingStore offsetStore; + + private SourceTask task; + private final Transformations transformations; + + private EmbeddedEngine(Configuration config, ClassLoader classLoader, Clock clock, DebeziumEngine.ChangeConsumer handler, + DebeziumEngine.CompletionCallback completionCallback, DebeziumEngine.ConnectorCallback connectorCallback, + OffsetCommitPolicy offsetCommitPolicy, SourceConnector sourceConnector, OffsetBackingStore offsetStore) { + this.connector = sourceConnector; + this.offsetStore = offsetStore; + this.config = config; + this.handler = handler; + this.classLoader = classLoader; + this.clock = clock; + this.completionCallback = completionCallback != null ? completionCallback : (success, msg, error) -> { + if (!success) { + LOGGER.error(msg, error); + } + }; + this.connectorCallback = connectorCallback; + this.completionResult = new CompletionResult(); + this.offsetCommitPolicy = offsetCommitPolicy; + + assert this.config != null; + assert this.handler != null; + assert this.classLoader != null; + assert this.clock != null; + keyConverter = config.getInstance(INTERNAL_KEY_CONVERTER_CLASS, Converter.class, () -> this.classLoader); + keyConverter.configure(config.subset(INTERNAL_KEY_CONVERTER_CLASS.name() + ".", true).asMap(), true); + valueConverter = config.getInstance(INTERNAL_VALUE_CONVERTER_CLASS, Converter.class, () -> this.classLoader); + Configuration valueConverterConfig = config; + if (valueConverter instanceof JsonConverter) { + // Make sure that the JSON converter is configured to NOT enable schemas ... + valueConverterConfig = config.edit().with(INTERNAL_VALUE_CONVERTER_CLASS + ".schemas.enable", false).build(); + } + valueConverter.configure(valueConverterConfig.subset(INTERNAL_VALUE_CONVERTER_CLASS.name() + ".", true).asMap(), false); + + transformations = new Transformations(config); + + // Create the worker config, adding extra fields that are required for validation of a worker config + // but that are not used within the embedded engine (since the source records are never serialized) ... + Map embeddedConfig = config.asMap(ALL_FIELDS); + embeddedConfig.put(WorkerConfig.KEY_CONVERTER_CLASS_CONFIG, JsonConverter.class.getName()); + embeddedConfig.put(WorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, JsonConverter.class.getName()); + workerConfig = new EmbeddedConfig(embeddedConfig); + } + + /** + * Determine if this embedded connector is currently running. + * + * @return {@code true} if running, or {@code false} otherwise + */ + public boolean isRunning() { + return this.runningThread.get() != null; + } + + private void fail(String msg) { + fail(msg, null); + } + + private void fail(String msg, Throwable error) { + if (completionResult.hasError()) { + // there's already a recorded failure, so keep the original one and simply log this one + LOGGER.error(msg, error); + return; + } + // don't use the completion callback here because we want to store the error and message only + completionResult.handle(false, msg, error); + } + + private void succeed(String msg) { + // don't use the completion callback here because we want to store the error and message only + completionResult.handle(true, msg, null); + } + + /** + * Run this embedded connector and deliver database changes to the registered {@link Consumer}. This method blocks until + * the connector is stopped. + *

+ * First, the method checks to see if this instance is currently {@link #run() running}, and if so immediately returns. + *

+ * If the configuration is valid, this method starts the connector and starts polling the connector for change events. + * All messages are delivered in batches to the {@link Consumer} registered with this embedded connector. The batch size, + * polling + * frequency, and other parameters are controlled via configuration settings. This continues until this connector is + * {@link #stop() stopped}. + *

+ * Note that there are two ways to stop a connector running on a thread: calling {@link #stop()} from another thread, or + * interrupting the thread (e.g., via {@link ExecutorService#shutdownNow()}). + *

+ * This method can be called repeatedly as needed. + */ + @Override + public void run() { + if (runningThread.compareAndSet(null, Thread.currentThread())) { + + final String engineName = config.getString(ENGINE_NAME); + final String connectorClassName = config.getString(CONNECTOR_CLASS); + final Optional connectorCallback = Optional.ofNullable(this.connectorCallback); + // Only one thread can be in this part of the method at a time ... + latch.countUp(); + try { + if (!config.validateAndRecord(CONNECTOR_FIELDS, LOGGER::error)) { + fail("Failed to start connector with invalid configuration (see logs for actual errors)"); + return; + } + + // Instantiate the connector ... + SourceConnector connector = null; + try { + @SuppressWarnings("unchecked") + Class connectorClass = (Class) classLoader.loadClass(connectorClassName); + connector = connectorClass.getDeclaredConstructor().newInstance(); + } + catch (Throwable t) { + fail("Unable to instantiate connector class '" + connectorClassName + "'", t); + return; + } + + //// Instantiate the offset store ... + //final String offsetStoreClassName = config.getString(OFFSET_STORAGE); + //OffsetBackingStore offsetStore = null; + //try { + // @SuppressWarnings("unchecked") + // Class offsetStoreClass = (Class) classLoader.loadClass(offsetStoreClassName); + // offsetStore = offsetStoreClass.getDeclaredConstructor().newInstance(); + //} + //catch (Throwable t) { + // fail("Unable to instantiate OffsetBackingStore class '" + offsetStoreClassName + "'", t); + // return; + //} + + // Initialize the offset store ... + try { + offsetStore.configure(workerConfig); + offsetStore.start(); + } + catch (Throwable t) { + fail("Unable to configure and start the '" + offsetStore.getClass().getName() + "' offset backing store", t); + return; + } + + // Set up the offset commit policy ... + if (offsetCommitPolicy == null) { + offsetCommitPolicy = Instantiator.getInstanceWithProperties(config.getString(EmbeddedEngine.OFFSET_COMMIT_POLICY), + () -> getClass().getClassLoader(), config.asProperties()); + } + + // Initialize the connector using a context that does NOT respond to requests to reconfigure tasks ... + ConnectorContext context = new ConnectorContext() { + + @Override + public void requestTaskReconfiguration() { + // Do nothing ... + } + + @Override + public void raiseError(Exception e) { + fail(e.getMessage(), e); + } + + }; + connector.initialize(context); + OffsetStorageWriter offsetWriter = new OffsetStorageWriter(offsetStore, engineName, + keyConverter, valueConverter); + OffsetStorageReader offsetReader = new OffsetStorageReaderImpl(offsetStore, engineName, + keyConverter, valueConverter); + Duration commitTimeout = Duration.ofMillis(config.getLong(OFFSET_COMMIT_TIMEOUT_MS)); + + try { + // Start the connector with the given properties and get the task configurations ... + connector.start(config.asMap()); + connectorCallback.ifPresent(DebeziumEngine.ConnectorCallback::connectorStarted); + List> taskConfigs = connector.taskConfigs(1); + Class taskClass = connector.taskClass(); + if (taskConfigs.isEmpty()) { + String msg = "Unable to start connector's task class '" + taskClass.getName() + "' with no task configuration"; + fail(msg); + return; + } + task = null; + try { + task = (SourceTask) taskClass.getDeclaredConstructor().newInstance(); + } + catch (IllegalAccessException | InstantiationException t) { + fail("Unable to instantiate connector's task class '" + taskClass.getName() + "'", t); + return; + } + try { + SourceTaskContext taskContext = new SourceTaskContext() { + @Override + public OffsetStorageReader offsetStorageReader() { + return offsetReader; + } + + // Purposely not marking this method with @Override as it was introduced in Kafka 2.x + // and otherwise would break builds based on Kafka 1.x + public Map configs() { + // TODO Auto-generated method stub + return null; + } + }; + task.initialize(taskContext); + task.start(taskConfigs.get(0)); + connectorCallback.ifPresent(DebeziumEngine.ConnectorCallback::taskStarted); + } + catch (Throwable t) { + // Mask the passwords ... + Configuration config = Configuration.from(taskConfigs.get(0)).withMaskedPasswords(); + String msg = "Unable to initialize and start connector's task class '" + taskClass.getName() + "' with config: " + + config; + fail(msg, t); + return; + } + + recordsSinceLastCommit = 0; + Throwable handlerError = null; + try { + timeOfLastCommitMillis = clock.currentTimeInMillis(); + RecordCommitter committer = buildRecordCommitter(offsetWriter, task, commitTimeout); + while (runningThread.get() != null) { + List changeRecords = null; + try { + LOGGER.debug("Embedded engine is polling task for records on thread {}", runningThread.get()); + changeRecords = task.poll(); // blocks until there are values ... + LOGGER.debug("Embedded engine returned from polling task for records"); + } + catch (InterruptedException e) { + // Interrupted while polling ... + LOGGER.debug("Embedded engine interrupted on thread {} while polling the task for records", runningThread.get()); + if (this.runningThread.get() == Thread.currentThread()) { + // this thread is still set as the running thread -> we were not interrupted + // due the stop() call -> probably someone else called the interrupt on us -> + // -> we should raise the interrupt flag + Thread.currentThread().interrupt(); + } + break; + } + catch (RetriableException e) { + LOGGER.info("Retrieable exception thrown, connector will be restarted", e); + // Retriable exception should be ignored by the engine + // and no change records delivered. + // The retry is handled in io.debezium.connector.common.BaseSourceTask.poll() + } + try { + if (changeRecords != null && !changeRecords.isEmpty()) { + LOGGER.debug("Received {} records from the task", changeRecords.size()); + changeRecords = changeRecords.stream() + .map(transformations::transform) + .filter(x -> x != null) + .collect(Collectors.toList()); + } + + if (changeRecords != null && !changeRecords.isEmpty()) { + LOGGER.debug("Received {} transformed records from the task", changeRecords.size()); + + try { + handler.handleBatch(changeRecords, committer); + } + catch (StopConnectorException e) { + break; + } + } + else { + LOGGER.debug("Received no records from the task"); + } + } + catch (Throwable t) { + // There was some sort of unexpected exception, so we should stop work + handlerError = t; + break; + } + } + } + finally { + if (handlerError != null) { + // There was an error in the handler so make sure it's always captured... + fail("Stopping connector after error in the application's handler method: " + handlerError.getMessage(), + handlerError); + } + try { + // First stop the task ... + LOGGER.debug("Stopping the task and engine"); + task.stop(); + connectorCallback.ifPresent(DebeziumEngine.ConnectorCallback::taskStopped); + // Always commit offsets that were captured from the source records we actually processed ... + commitOffsets(offsetWriter, commitTimeout, task); + if (handlerError == null) { + // We stopped normally ... + succeed("Connector '" + connectorClassName + "' completed normally."); + } + } + catch (Throwable t) { + fail("Error while trying to stop the task and commit the offsets", t); + } + } + } + catch (Throwable t) { + fail("Error while trying to run connector class '" + connectorClassName + "'", t); + } + finally { + // Close the offset storage and finally the connector ... + try { + offsetStore.stop(); + } + catch (Throwable t) { + fail("Error while trying to stop the offset store", t); + } + finally { + try { + connector.stop(); + connectorCallback.ifPresent(DebeziumEngine.ConnectorCallback::connectorStopped); + } + catch (Throwable t) { + fail("Error while trying to stop connector class '" + connectorClassName + "'", t); + } + } + } + } + finally { + latch.countDown(); + runningThread.set(null); + // after we've "shut down" the engine, fire the completion callback based on the results we collected + completionCallback.handle(completionResult.success(), completionResult.message(), completionResult.error()); + } + } + } + + /** + * Creates a new RecordCommitter that is responsible for informing the engine + * about the updates to the given batch + * @param offsetWriter the offsetWriter current in use + * @param task the sourcetask + * @param commitTimeout the time in ms until a commit times out + * @return the new recordCommitter to be used for a given batch + */ + protected RecordCommitter buildRecordCommitter(OffsetStorageWriter offsetWriter, SourceTask task, Duration commitTimeout) { + return new RecordCommitter() { + + @Override + public synchronized void markProcessed(SourceRecord record) throws InterruptedException { + task.commitRecord(record); + recordsSinceLastCommit += 1; + offsetWriter.offset(record.sourcePartition(), record.sourceOffset()); + } + + @Override + public synchronized void markBatchFinished() { + maybeFlush(offsetWriter, offsetCommitPolicy, commitTimeout, task); + } + }; + } + + /** + * Determine if we should flush offsets to storage, and if so then attempt to flush offsets. + * + * @param offsetWriter the offset storage writer; may not be null + * @param policy the offset commit policy; may not be null + * @param commitTimeout the timeout to wait for commit results + * @param task the task which produced the records for which the offsets have been committed + */ + protected void maybeFlush(OffsetStorageWriter offsetWriter, OffsetCommitPolicy policy, Duration commitTimeout, + SourceTask task) { + // Determine if we need to commit to offset storage ... + long timeSinceLastCommitMillis = clock.currentTimeInMillis() - timeOfLastCommitMillis; + if (policy.performCommit(recordsSinceLastCommit, Duration.ofMillis(timeSinceLastCommitMillis))) { + commitOffsets(offsetWriter, commitTimeout, task); + } + } + + /** + * Flush offsets to storage. + * + * @param offsetWriter the offset storage writer; may not be null + * @param commitTimeout the timeout to wait for commit results + * @param task the task which produced the records for which the offsets have been committed + */ + protected void commitOffsets(OffsetStorageWriter offsetWriter, Duration commitTimeout, SourceTask task) { + long started = clock.currentTimeInMillis(); + long timeout = started + commitTimeout.toMillis(); + if (!offsetWriter.beginFlush()) { + return; + } + Future flush = offsetWriter.doFlush(this::completedFlush); + if (flush == null) { + return; // no offsets to commit ... + } + + // Wait until the offsets are flushed ... + try { + flush.get(Math.max(timeout - clock.currentTimeInMillis(), 0), TimeUnit.MILLISECONDS); + // if we've gotten this far, the offsets have been committed so notify the task + task.commit(); + recordsSinceLastCommit = 0; + timeOfLastCommitMillis = clock.currentTimeInMillis(); + } + catch (InterruptedException e) { + LOGGER.warn("Flush of {} offsets interrupted, cancelling", this); + offsetWriter.cancelFlush(); + } + catch (ExecutionException e) { + LOGGER.error("Flush of {} offsets threw an unexpected exception: ", this, e); + offsetWriter.cancelFlush(); + } + catch (TimeoutException e) { + LOGGER.error("Timed out waiting to flush {} offsets to storage", this); + offsetWriter.cancelFlush(); + } + } + + protected void completedFlush(Throwable error, Void result) { + if (error != null) { + LOGGER.error("Failed to flush {} offsets to storage: ", this, error); + } + else { + LOGGER.trace("Finished flushing {} offsets to storage", this); + } + } + + /** + * Stop the execution of this embedded connector. This method does not block until the connector is stopped; use + * {@link #await(long, TimeUnit)} for this purpose. + * + * @return {@code true} if the connector was {@link #run() running} and will eventually stop, or {@code false} if it was not + * running when this method is called + * @see #await(long, TimeUnit) + */ + public boolean stop() { + LOGGER.info("Stopping the embedded engine"); + // Signal that the run() method should stop ... + Thread thread = this.runningThread.getAndSet(null); + if (thread != null) { + try { + // Making sure the event source coordinator has enough time to shut down before forcefully stopping it + Duration timeout = Duration.ofMillis(Long + .valueOf(System.getProperty(WAIT_FOR_COMPLETION_BEFORE_INTERRUPT_PROP, Long.toString(WAIT_FOR_COMPLETION_BEFORE_INTERRUPT_DEFAULT.toMillis())))); + LOGGER.info("Waiting for {} for connector to stop", timeout); + latch.await(timeout.toMillis(), TimeUnit.MILLISECONDS); + } + catch (InterruptedException e) { + } + LOGGER.debug("Interrupting the embedded engine's thread {} (already interrupted: {})", thread, thread.isInterrupted()); + // Interrupt the thread in case it is blocked while polling the task for records ... + thread.interrupt(); + return true; + } + return false; + } + + @Override + public void close() throws IOException { + stop(); + } + + /** + * Wait for the connector to complete processing. If the processor is not running, this method returns immediately; however, + * if the processor is {@link #stop() stopped} and restarted before this method is called, this method will return only + * when it completes the second time. + * + * @param timeout the maximum amount of time to wait before returning + * @param unit the unit of time; may not be null + * @return {@code true} if the connector completed within the timeout (or was not running), or {@code false} if it is still + * running when the timeout occurred + * @throws InterruptedException if this thread is interrupted while waiting for the completion of the connector + */ + public boolean await(long timeout, TimeUnit unit) throws InterruptedException { + return latch.await(timeout, unit); + } + + @Override + public String toString() { + return "EmbeddedEngine{id=" + config.getString(ENGINE_NAME) + '}'; + } + + public void runWithTask(Consumer consumer) { + consumer.accept(task); + } + + protected static class EmbeddedConfig extends WorkerConfig { + private static final ConfigDef CONFIG; + + static { + ConfigDef config = baseConfigDef(); + Field.group(config, "file", OFFSET_STORAGE_FILE_FILENAME); + Field.group(config, "kafka", OFFSET_STORAGE_KAFKA_TOPIC); + Field.group(config, "kafka", OFFSET_STORAGE_KAFKA_PARTITIONS); + Field.group(config, "kafka", OFFSET_STORAGE_KAFKA_REPLICATION_FACTOR); + CONFIG = config; + } + + protected EmbeddedConfig(Map props) { + super(CONFIG, props); + } + } +} diff --git a/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/EmbeddedEngineExecutorService.java b/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/EmbeddedEngineExecutorService.java new file mode 100644 index 00000000..e7f1fad6 --- /dev/null +++ b/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/EmbeddedEngineExecutorService.java @@ -0,0 +1,52 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.common.cdc; + +import java.io.Closeable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * @author Christian Tzolov + */ +public class EmbeddedEngineExecutorService implements Closeable { + + private static final Log logger = LogFactory.getLog(EmbeddedEngineExecutorService.class); + + private final EmbeddedEngine engine; + private final ExecutorService executor; + + public EmbeddedEngineExecutorService(EmbeddedEngine engine) { + this.engine = engine; + this.executor = Executors.newSingleThreadExecutor(); + } + + public void start() { + logger.info("Start Embedded Engine"); + this.executor.execute(this.engine); + } + + @Override + public void close() { + logger.info("Stop Embedded Engine"); + this.engine.stop(); + this.executor.shutdown(); + } +} diff --git a/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/store/MetadataStoreOffsetBackingStore.java b/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/store/MetadataStoreOffsetBackingStore.java new file mode 100644 index 00000000..d75dc505 --- /dev/null +++ b/functions/common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/store/MetadataStoreOffsetBackingStore.java @@ -0,0 +1,90 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.common.cdc.store; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.kafka.connect.runtime.WorkerConfig; +import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; +import org.apache.kafka.connect.storage.MemoryOffsetBackingStore; + +import org.springframework.integration.metadata.MetadataStore; + +/** + * @author Christian Tzolov + */ +public class MetadataStoreOffsetBackingStore extends MemoryOffsetBackingStore { + + private MetadataStore metadataStore; + private String offsetStoreName; + + public MetadataStoreOffsetBackingStore(MetadataStore metadataStore) { + this.metadataStore = metadataStore; + } + + @Override + public void configure(WorkerConfig config) { + super.configure(config); + this.offsetStoreName = config.getString(StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG); + } + + @Override + public synchronized void start() { + super.start(); + //log.info("Starting FileOffsetBackingStore with file {}", file); + load(); + } + + @Override + public synchronized void stop() { + super.stop(); + // Nothing to do since this doesn't maintain any outstanding connections/data + // log.info("Stopped FileOffsetBackingStore"); + } + + private void load() { + if (this.metadataStore.get("keys") != null) { + String[] keys = this.metadataStore.get("keys").split(","); + for (String keySting : keys) { + String valueString = this.metadataStore.get(keySting); + ByteBuffer key = ByteBuffer.wrap(keySting.getBytes()); + ByteBuffer value = ByteBuffer.wrap(valueString.getBytes()); + this.data.put(key, value); + } + } + } + + @Override + protected void save() { + List keys = new ArrayList<>(); + for (Map.Entry mapEntry : this.data.entrySet()) { + byte[] key = (mapEntry.getKey() != null) ? mapEntry.getKey().array() : null; + byte[] value = (mapEntry.getValue() != null) ? mapEntry.getValue().array() : null; + if (key != null && value != null) { + keys.add(new String(key, StandardCharsets.UTF_8)); + this.metadataStore.put( + new String(key, StandardCharsets.UTF_8), + new String(value, StandardCharsets.UTF_8)); + } + } + this.metadataStore.put("keys", String.join(",", keys)); + } +} diff --git a/functions/common/cdc-debezium-common/src/main/resources/application.properties b/functions/common/cdc-debezium-common/src/main/resources/application.properties new file mode 100644 index 00000000..8fb76f9d --- /dev/null +++ b/functions/common/cdc-debezium-common/src/main/resources/application.properties @@ -0,0 +1,2 @@ +spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration +spring.cloud.stream.kafka.default.producer.messageKeyExpression=headers['cdc_key'] diff --git a/functions/function-dependencies/pom.xml b/functions/function-dependencies/pom.xml index 4f5a7b03..6b4a0d6b 100644 --- a/functions/function-dependencies/pom.xml +++ b/functions/function-dependencies/pom.xml @@ -15,6 +15,11 @@ + + org.springframework.cloud.fn + cdc-debezium-supplier + ${project.version} + org.springframework.cloud.fn file-supplier @@ -244,4 +249,4 @@ - \ No newline at end of file + diff --git a/functions/pom.xml b/functions/pom.xml index b1af83ed..b53fcb19 100644 --- a/functions/pom.xml +++ b/functions/pom.xml @@ -51,6 +51,8 @@ common/tcp-common common/twitter-common common/tensorflow-common + common/cdc-debezium-common + common/cdc-debezium-boot-starter consumer/cassandra-consumer consumer/analytics-consumer @@ -98,6 +100,7 @@ supplier/websocket-supplier supplier/s3-supplier supplier/twitter-supplier + supplier/cdc-debezium-supplier spring-functions-parent function-dependencies diff --git a/functions/spring-functions-parent/pom.xml b/functions/spring-functions-parent/pom.xml index 0be98eb3..f3e16f48 100644 --- a/functions/spring-functions-parent/pom.xml +++ b/functions/spring-functions-parent/pom.xml @@ -38,4 +38,4 @@ - \ No newline at end of file + diff --git a/functions/supplier/cdc-debezium-supplier/README.adoc b/functions/supplier/cdc-debezium-supplier/README.adoc new file mode 100644 index 00000000..9ad6dab5 --- /dev/null +++ b/functions/supplier/cdc-debezium-supplier/README.adoc @@ -0,0 +1,45 @@ +# CDC Debezium Supplier + +This module provides a CDC Debezium supplier that can be reused and composed in other applications. +The `Supplier` uses the `Debezium` library. +`cdcSupplier` is implemented as a `java.util.function.Supplier`. +This supplier gives you a reactive stream from CDC sources. The supplier has a signature of `Supplier>>`. +Users have to subscribe to this `Flux` and then receive the data. + +https://en.wikipedia.org/wiki/Change_data_capture[Change Data Capture] (CDC) `source` that captures and streams change events from various databases. +Currently, it supports `MySQL`, `PostgreSQL`, `MongoDB`, `Oracle` and `SQL Server` databases. + +Build upon https://debezium.io/docs/embedded/[Debezium Embedded Connector], the `CDC Source` allows capturing and streaming database changes over different message binders such Apache Kafka, RabbitMQ and all Spring Cloud Stream supporter brokers. + +## Beans for injection + +You can import the `CdcSupplierConfiguration` in the application and then inject the following bean. + +`cdcSupplier` + +You need to inject this as `Supplier>>`. + +You can use `cdcSupplier` as a qualifier when injecting. + +Once injected, you can use the `get` method of the `Supplier` to invoke it and then subscribe to the returned `Flux`. + +## Configuration Options + +All configuration properties are prefixed with `cdc`. +Use the `cdc.config.` prefix to inject native, Debezium properties. Convenient shortcuts for the most frequently used +Debezium properties are available. +Uses `BackingOffsetStore` configuration, based on the [MetadataStore](https://github.com/spring-cloud/stream-applications/tree/master/functions/common/metadata-store-common) service. +Later provides various microservices friendly ways for storing the offset metadata. + +For more information on the various options available, please see: + * link:../../common/cdc-debezium-common/src/main/java/org/springframework/cloud/fn/common/cdc/CdcCommonProperties.java[CdcCommonProperties]. + * link:src/main/java/org/springframework/cloud/fn/supplier/cdc-debezium-supplier/CdcSupplierProperties.java[CdcSupplierProperties]. + +## Examples +See this link:../../../applications/source/cdc-debezium-source/src/test/java/org/springframework/cloud/stream/app/source/cdc[test suite] for the various ways, this supplier is used. + +## Other usage + +See this link:../../../applications/source/cdc-debezium-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream CDC Source. + +Also check link:../../common/cdc-debezium-boot-starter/README.adoc[README] to see how the core CDC can be used a Spring Boot Starter. diff --git a/functions/supplier/cdc-debezium-supplier/pom.xml b/functions/supplier/cdc-debezium-supplier/pom.xml new file mode 100644 index 00000000..192fabcf --- /dev/null +++ b/functions/supplier/cdc-debezium-supplier/pom.xml @@ -0,0 +1,124 @@ + + + 4.0.0 + cdc-debezium-supplier + 1.0.0-SNAPSHOT + cdc-debezium-supplier + CDC Debezium Suppliers + + + org.springframework.cloud.fn + spring-functions-parent + 1.0.0-SNAPSHOT + ../../spring-functions-parent + + + + 1.2.1.Final + 8.0.13 + + + + + org.springframework.cloud.fn + cdc-debezium-common + ${project.version} + + + mysql + mysql-connector-java + ${mysql.version} + + + + + io.debezium + debezium-connector-mysql + + + mysql-connector-java + mysql + + + ${version.debezium} + + + + io.debezium + debezium-connector-mongodb + + + slf4j-log4j12 + org.slf4j + + + ${version.debezium} + + + io.debezium + debezium-connector-postgres + + + slf4j-log4j12 + org.slf4j + + + ${version.debezium} + + + io.debezium + debezium-connector-oracle + + + slf4j-log4j12 + org.slf4j + + + ${version.debezium} + + + io.debezium + debezium-connector-sqlserver + + + slf4j-log4j12 + org.slf4j + + + ${version.debezium} + + + + + org.springframework.boot + spring-boot-starter-integration + + + + org.springframework.boot + spring-boot-starter-validation + + + org.springframework.boot + spring-boot-configuration-processor + provided + + + + org.springframework.boot + spring-boot-starter-test + test + + + io.projectreactor + reactor-test + test + + + org.springframework.integration + spring-integration-test + test + + + + diff --git a/functions/supplier/cdc-debezium-supplier/src/main/java/org/springframework/cloud/fn/supplier/cdc/CdcSupplierConfiguration.java b/functions/supplier/cdc-debezium-supplier/src/main/java/org/springframework/cloud/fn/supplier/cdc/CdcSupplierConfiguration.java new file mode 100644 index 00000000..c2a6c60a --- /dev/null +++ b/functions/supplier/cdc-debezium-supplier/src/main/java/org/springframework/cloud/fn/supplier/cdc/CdcSupplierConfiguration.java @@ -0,0 +1,171 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.cdc; + +import java.lang.reflect.Field; +import java.util.Iterator; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.kafka.connect.header.Header; +import org.apache.kafka.connect.source.SourceRecord; +import reactor.core.publisher.EmitterProcessor; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; + +import org.springframework.beans.factory.BeanClassLoaderAware; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.fn.common.cdc.CdcCommonConfiguration; +import org.springframework.cloud.fn.common.cdc.EmbeddedEngine; +import org.springframework.cloud.fn.common.cdc.EmbeddedEngineExecutorService; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.util.ClassUtils; +import org.springframework.util.MimeTypeUtils; + +/** + * CDC source that uses the Debezium Connectors to monitor and record all of the row-level changes in the databases. + * https://debezium.io/docs/connectors + * + * @author Christian Tzolov + */ +@EnableConfigurationProperties(CdcSupplierProperties.class) +@Import(CdcCommonConfiguration.class) +public class CdcSupplierConfiguration implements BeanClassLoaderAware { + + private static final Log logger = LogFactory.getLog(CdcSupplierConfiguration.class); + + /** + * ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL. + */ + public static final String ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL = "org.springframework.kafka.support.KafkaNull"; + + private Object kafkaNull = null; + + @Override + public void setBeanClassLoader(ClassLoader classLoader) { + try { + Class clazz = ClassUtils.forName(ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL, classLoader); + Field field = clazz.getDeclaredField("INSTANCE"); + this.kafkaNull = field.get(null); + } + catch (ClassNotFoundException | NoSuchFieldException | IllegalAccessException e) { + } + } + + @Bean + public Supplier>> cdcSupplier(EmbeddedEngineExecutorService embeddedEngineExecutorService) { + + return () -> emitterProcessor + .doOnSubscribe(subscription -> { + embeddedEngineExecutorService.start(); + }) + .doAfterTerminate(() -> { + logger.info("Proactive shutdown"); + embeddedEngineExecutorService.close(); + }) + .doOnError(throwable -> logger.error(throwable.getMessage(), throwable)); + } + + private EmitterProcessor> emitterProcessor = EmitterProcessor.create(1, false); + + @Bean + public EmbeddedEngineExecutorService embeddedEngineExecutorService( + EmbeddedEngine.Builder embeddedEngineBuilder, + Function valueSerializer, Function keySerializer, + Function recordFlattering, + ObjectMapper mapper, CdcSupplierProperties cdcStreamingEngineProperties) { + + FluxSink> sink = emitterProcessor.sink(); + Consumer messageConsumer = sourceRecord -> { + + // When cdc.flattering.deleteHandlingMode=none and cdc.flattering.dropTombstones=false + // then on deletion event an additional sourceRecord is sent with value Null. + // Here we filter out such condition. + if (sourceRecord == null) { + logger.debug("Ignore disabled tombstone events"); + return; + } + + Object cdcJsonPayload = valueSerializer.apply(sourceRecord); + + // When the tombstone event is enabled, Debezium serializes the payload to null (e.g. empty payload) + // while the metadata information is carried through the headers (cdc_key). + // Note: Event for none flattered responses, when the cdc.config.tombstones.on.delete=true (default), + // tombstones are generate by Debezium and handled by the code below. + if (cdcJsonPayload == null) { + cdcJsonPayload = this.kafkaNull; + } + + // If payload is still null ignore the message. + if (cdcJsonPayload == null) { + logger.info("dropped null payload message"); + return; + } + + byte[] key = keySerializer.apply(sourceRecord); + if (key == null) { + logger.warn("Null serialised key for sourceRecord: " + sourceRecord); + key = new byte[0]; + } + + MessageBuilder messageBuilder = MessageBuilder + .withPayload(cdcJsonPayload) + .setHeader("cdc_key", new String(key)) + .setHeader("cdc_topic", sourceRecord.topic()) + .setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON_VALUE); + + if (cdcStreamingEngineProperties.getHeader().isConvertConnectHeaders()) { + // Convert the Connect Headers into Message Headers. + if (sourceRecord.headers() != null && !sourceRecord.headers().isEmpty()) { + Iterator

itr = sourceRecord.headers().iterator(); + while (itr.hasNext()) { + Header header = itr.next(); + messageBuilder.setHeader(header.key(), header.value()); + } + } + } + + if (cdcStreamingEngineProperties.getHeader().isOffset()) { + try { + messageBuilder.setHeader("cdc_offset", + mapper.writeValueAsString(sourceRecord.sourceOffset())); + } + catch (JsonProcessingException e) { + logger.warn("Failed to record cdc_offset header", e); + } + } + + + sink.next(messageBuilder.build()); + }; + + EmbeddedEngine engine = embeddedEngineBuilder + .notifying(record -> messageConsumer.accept(recordFlattering.apply(record))) + .build(); + + return new EmbeddedEngineExecutorService(engine); + } +} diff --git a/functions/supplier/cdc-debezium-supplier/src/main/java/org/springframework/cloud/fn/supplier/cdc/CdcSupplierProperties.java b/functions/supplier/cdc-debezium-supplier/src/main/java/org/springframework/cloud/fn/supplier/cdc/CdcSupplierProperties.java new file mode 100644 index 00000000..8aa3a071 --- /dev/null +++ b/functions/supplier/cdc-debezium-supplier/src/main/java/org/springframework/cloud/fn/supplier/cdc/CdcSupplierProperties.java @@ -0,0 +1,68 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.supplier.cdc; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +/** + * + * @author Christian Tzolov + */ +@ConfigurationProperties("cdc.stream") +@Validated +public class CdcSupplierProperties { + + /** + * Control what information to be serialized in the outbound message headers. + */ + private Header header = new Header(); + + public Header getHeader() { + return header; + } + + public static class Header { + + /** + * Serializes the source record's offset metadata into the outbound message header under cdc.offset. + */ + private boolean offset = false; + + /** + * When true the {@link org.apache.kafka.connect.header.Header} are converted into message headers with the + * {@link org.apache.kafka.connect.header.Header#key()} as name and {@link org.apache.kafka.connect.header.Header#value()}. + */ + private boolean convertConnectHeaders = true; + + public boolean isOffset() { + return offset; + } + + public void setOffset(boolean offset) { + this.offset = offset; + } + + public boolean isConvertConnectHeaders() { + return convertConnectHeaders; + } + + public void setConvertConnectHeaders(boolean convertConnectHeaders) { + this.convertConnectHeaders = convertConnectHeaders; + } + } +} diff --git a/functions/supplier/cdc-debezium-supplier/src/main/resources/application.properties b/functions/supplier/cdc-debezium-supplier/src/main/resources/application.properties new file mode 100644 index 00000000..8fb76f9d --- /dev/null +++ b/functions/supplier/cdc-debezium-supplier/src/main/resources/application.properties @@ -0,0 +1,2 @@ +spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration +spring.cloud.stream.kafka.default.producer.messageKeyExpression=headers['cdc_key']