From e66741c7c677c5864bda7936298ccf879e62fcaa Mon Sep 17 00:00:00 2001 From: Artem Bilan Date: Tue, 17 Sep 2024 15:22:08 -0400 Subject: [PATCH] GH-9481: Fix `JdbcMetadataStore` for PostgreSQL & MySQL Fixes: #9481 Issue link: https://github.com/spring-projects/spring-integration/issues/9481 MySQL throws `CannotAcquireLockException` in case of duplicate key failure. PostgreSQL just rollbacks a transaction not letting us move on with a `SELECT` * Include `TransientDataAccessException` to the catch block of the `INSERT` to ignore it for the subsequent `SELECT` * Add logic to determine `PostgreSQL` database vendor and include `ON CONFLICT DO NOTHING` hint into the `INSERT` to not fail in case of duplicate key found on `putIfAbsent` operation (cherry picked from commit 98d0266ba06680e035acb04306f51398c2b474e1) --- .../jdbc/metadata/JdbcMetadataStore.java | 10 +- .../jdbc/mysql/MySqlMetadataStoreTests.java | 103 ++++++++++++++++++ ...resChannelMessageTableSubscriberTests.java | 5 +- .../PostgresContainerTest.java | 4 +- .../postgres/PostgresMetadataStoreTests.java | 93 ++++++++++++++++ .../PostgresJdbcChannelMessageStoreTests.java | 4 +- 6 files changed, 213 insertions(+), 6 deletions(-) create mode 100644 spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/mysql/MySqlMetadataStoreTests.java rename spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/{channel => postgres}/PostgresChannelMessageTableSubscriberTests.java (97%) rename spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/{channel => postgres}/PostgresContainerTest.java (94%) create mode 100644 spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/postgres/PostgresMetadataStoreTests.java diff --git a/spring-integration-jdbc/src/main/java/org/springframework/integration/jdbc/metadata/JdbcMetadataStore.java b/spring-integration-jdbc/src/main/java/org/springframework/integration/jdbc/metadata/JdbcMetadataStore.java index 2faca2a224..ec03945c11 100644 --- a/spring-integration-jdbc/src/main/java/org/springframework/integration/jdbc/metadata/JdbcMetadataStore.java +++ b/spring-integration-jdbc/src/main/java/org/springframework/integration/jdbc/metadata/JdbcMetadataStore.java @@ -25,7 +25,9 @@ import org.springframework.context.SmartLifecycle; import org.springframework.core.log.LogAccessor; import org.springframework.dao.DataIntegrityViolationException; import org.springframework.dao.EmptyResultDataAccessException; +import org.springframework.dao.TransientDataAccessException; import org.springframework.integration.metadata.ConcurrentMetadataStore; +import org.springframework.jdbc.core.ConnectionCallback; import org.springframework.jdbc.core.JdbcOperations; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.transaction.annotation.Transactional; @@ -168,12 +170,18 @@ public class JdbcMetadataStore implements ConcurrentMetadataStore, InitializingB @Override public void afterPropertiesSet() { + String dataBaseVendor = + this.jdbcTemplate.execute((ConnectionCallback) connection -> + connection.getMetaData().getDatabaseProductName()); this.getValueQuery = String.format(this.getValueQuery, this.tablePrefix); this.getValueForUpdateQuery = String.format(this.getValueForUpdateQuery, this.tablePrefix, this.lockHint); this.replaceValueQuery = String.format(this.replaceValueQuery, this.tablePrefix); this.replaceValueByKeyQuery = String.format(this.replaceValueByKeyQuery, this.tablePrefix); this.removeValueQuery = String.format(this.removeValueQuery, this.tablePrefix); this.putIfAbsentValueQuery = String.format(this.putIfAbsentValueQuery, this.tablePrefix, this.tablePrefix); + if ("PostgreSQL".equals(dataBaseVendor)) { + this.putIfAbsentValueQuery += " ON CONFLICT DO NOTHING"; + } this.countQuery = String.format(this.countQuery, this.tablePrefix); } @@ -247,7 +255,7 @@ public class JdbcMetadataStore implements ConcurrentMetadataStore, InitializingB ps.setString(5, this.region); // NOSONAR magic number }); } - catch (DataIntegrityViolationException ex) { + catch (TransientDataAccessException | DataIntegrityViolationException ex) { return 0; } } diff --git a/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/mysql/MySqlMetadataStoreTests.java b/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/mysql/MySqlMetadataStoreTests.java new file mode 100644 index 0000000000..978a4dbe1c --- /dev/null +++ b/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/mysql/MySqlMetadataStoreTests.java @@ -0,0 +1,103 @@ +/* + * Copyright 2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.jdbc.mysql; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import javax.sql.DataSource; + +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.Resource; +import org.springframework.integration.jdbc.metadata.JdbcMetadataStore; +import org.springframework.integration.metadata.ConcurrentMetadataStore; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.init.DataSourceInitializer; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Artem Bilan + * + * @since 6.4 + */ +@SpringJUnitConfig +@DirtiesContext +class MySqlMetadataStoreTests implements MySqlContainerTest { + + @Autowired + ConcurrentMetadataStore jdbcMetadataStore; + + @Test + void verifyJdbcMetadataStoreConcurrency() throws InterruptedException { + ExecutorService executorService = Executors.newFixedThreadPool(100); + CountDownLatch successPutIfAbsents = new CountDownLatch(100); + for (int i = 0; i < 100; i++) { + executorService.execute(() -> { + this.jdbcMetadataStore.putIfAbsent("testKey", "testValue"); + successPutIfAbsents.countDown(); + }); + } + assertThat(successPutIfAbsents.await(10, TimeUnit.SECONDS)).isTrue(); + executorService.shutdown(); + } + + @Configuration(proxyBeanMethods = false) + @EnableTransactionManagement + static class TestConfiguration { + + @Value("org/springframework/integration/jdbc/schema-mysql.sql") + Resource createSchemaScript; + + @Bean + DataSource dataSource() { + return MySqlContainerTest.dataSource(); + } + + @Bean + DataSourceInitializer dataSourceInitializer(DataSource dataSource) { + DataSourceInitializer dataSourceInitializer = new DataSourceInitializer(); + dataSourceInitializer.setDataSource(dataSource); + dataSourceInitializer.setDatabasePopulator(new ResourceDatabasePopulator(this.createSchemaScript)); + return dataSourceInitializer; + } + + @Bean + PlatformTransactionManager transactionManager(DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Bean + JdbcMetadataStore jdbcMetadataStore(DataSource dataSource) { + return new JdbcMetadataStore(dataSource); + } + + } + +} diff --git a/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/channel/PostgresChannelMessageTableSubscriberTests.java b/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/postgres/PostgresChannelMessageTableSubscriberTests.java similarity index 97% rename from spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/channel/PostgresChannelMessageTableSubscriberTests.java rename to spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/postgres/PostgresChannelMessageTableSubscriberTests.java index d028e7ee82..55f826081b 100644 --- a/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/channel/PostgresChannelMessageTableSubscriberTests.java +++ b/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/postgres/PostgresChannelMessageTableSubscriberTests.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.integration.jdbc.channel; +package org.springframework.integration.jdbc.postgres; import java.sql.DriverManager; import java.sql.SQLException; @@ -44,6 +44,9 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.io.ByteArrayResource; import org.springframework.integration.config.EnableIntegration; +import org.springframework.integration.jdbc.channel.PgConnectionSupplier; +import org.springframework.integration.jdbc.channel.PostgresChannelMessageTableSubscriber; +import org.springframework.integration.jdbc.channel.PostgresSubscribableChannel; import org.springframework.integration.jdbc.store.JdbcChannelMessageStore; import org.springframework.integration.jdbc.store.channel.PostgresChannelMessageStoreQueryProvider; import org.springframework.jdbc.datasource.DataSourceTransactionManager; diff --git a/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/channel/PostgresContainerTest.java b/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/postgres/PostgresContainerTest.java similarity index 94% rename from spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/channel/PostgresContainerTest.java rename to spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/postgres/PostgresContainerTest.java index d85167891b..71ef40d625 100644 --- a/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/channel/PostgresContainerTest.java +++ b/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/postgres/PostgresContainerTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2022-2023 the original author or authors. + * Copyright 2022-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.integration.jdbc.channel; +package org.springframework.integration.jdbc.postgres; import org.junit.jupiter.api.BeforeAll; import org.testcontainers.containers.PostgreSQLContainer; diff --git a/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/postgres/PostgresMetadataStoreTests.java b/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/postgres/PostgresMetadataStoreTests.java new file mode 100644 index 0000000000..f4ddb475c6 --- /dev/null +++ b/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/postgres/PostgresMetadataStoreTests.java @@ -0,0 +1,93 @@ +/* + * Copyright 2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.jdbc.postgres; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import javax.sql.DataSource; + +import org.apache.commons.dbcp2.BasicDataSource; +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.integration.jdbc.metadata.JdbcMetadataStore; +import org.springframework.integration.metadata.ConcurrentMetadataStore; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Artem Bilan + * + * @since 6.2.9 + */ +@SpringJUnitConfig +@DirtiesContext +class PostgresMetadataStoreTests implements PostgresContainerTest { + + @Autowired + ConcurrentMetadataStore jdbcMetadataStore; + + @Test + void verifyJdbcMetadataStoreConcurrency() throws InterruptedException { + ExecutorService executorService = Executors.newFixedThreadPool(100); + CountDownLatch successPutIfAbsents = new CountDownLatch(100); + for (int i = 0; i < 100; i++) { + executorService.execute(() -> { + this.jdbcMetadataStore.putIfAbsent("testKey", "testValue"); + successPutIfAbsents.countDown(); + }); + } + assertThat(successPutIfAbsents.await(10, TimeUnit.SECONDS)).isTrue(); + executorService.shutdown(); + } + + @Configuration(proxyBeanMethods = false) + @EnableTransactionManagement + static class TestConfiguration { + + @Bean + DataSource dataSource() { + BasicDataSource dataSource = new BasicDataSource(); + dataSource.setUrl(PostgresContainerTest.getJdbcUrl()); + dataSource.setUsername(PostgresContainerTest.getUsername()); + dataSource.setPassword(PostgresContainerTest.getPassword()); + return dataSource; + } + + @Bean + PlatformTransactionManager transactionManager(DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Bean + JdbcMetadataStore jdbcMetadataStore(DataSource dataSource) { + return new JdbcMetadataStore(dataSource); + } + + } + +} diff --git a/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/store/channel/PostgresJdbcChannelMessageStoreTests.java b/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/store/channel/PostgresJdbcChannelMessageStoreTests.java index 9a7835f0b5..b0b694a91c 100644 --- a/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/store/channel/PostgresJdbcChannelMessageStoreTests.java +++ b/spring-integration-jdbc/src/test/java/org/springframework/integration/jdbc/store/channel/PostgresJdbcChannelMessageStoreTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 the original author or authors. + * Copyright 2023-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ import org.apache.commons.dbcp2.BasicDataSource; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.integration.jdbc.channel.PostgresContainerTest; +import org.springframework.integration.jdbc.postgres.PostgresContainerTest; import org.springframework.jdbc.datasource.DataSourceTransactionManager; import org.springframework.test.context.ContextConfiguration; import org.springframework.transaction.PlatformTransactionManager;